From a2b66e943b29d9a7a8c6bb14fd569fb5b402f55c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 2 Oct 2019 22:45:12 -0700 Subject: [PATCH 01/12] fix: Big speed improvment when loading MISPEvent 1. `properties` is a list comprehension 2. Massively reduce the amount of calls to `properties` --- pymisp/abstract.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/pymisp/abstract.py b/pymisp/abstract.py index c85ef3b..39c6165 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -107,12 +107,7 @@ class AbstractMISP(MutableMapping): """All the class public properties that will be dumped in the dictionary, and the JSON export. Note: all the properties starting with a `_` (private), or listed in __not_jsonable will be skipped. """ - to_return = [] - for prop, value in vars(self).items(): - if prop.startswith('_') or prop in self.__not_jsonable: - continue - to_return.append(prop) - return to_return + return [k for k in vars(self).keys() if not (k[0] == '_' or k in self.__not_jsonable)] def from_dict(self, **kwargs): """Loading all the parameters as class properties, if they aren't `None`. @@ -216,8 +211,9 @@ class AbstractMISP(MutableMapping): raise Exception('edited can only be True or False') def __setattr__(self, name, value): - if name in self.properties: - self.__edited = True + if name != '_AbstractMISP__edited': + if not self.__edited and name in self.properties: + self.__edited = True super(AbstractMISP, self).__setattr__(name, value) def _datetime_to_timestamp(self, d): From e05c7d9b4ff723e5d6843d9ae54db9f4a76c5056 Mon Sep 17 00:00:00 2001 From: Marc Hoersken Date: Thu, 3 Oct 2019 19:12:19 +0200 Subject: [PATCH 02/12] Cache JSON definitions in memory LFU cache provided by cachetools - Path and modified time of JSON file are used as the cache key - Global state is hidden away inside a root-class for re-use - Maximum size is 150 considering the number of JSON definitions During my tests the memory usage of the test suites was halved. --- Pipfile.lock | 14 ++++++++++++++ pymisp/abstract.py | 33 ++++++++++++++++++++++++++++++++- pymisp/api.py | 15 +++------------ pymisp/aping.py | 3 +-- pymisp/mispevent.py | 30 +++++------------------------- setup.py | 2 +- 6 files changed, 56 insertions(+), 41 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 5eb4506..5a70930 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -31,6 +31,13 @@ ], "version": "==4.8.0" }, + "cachetools": { + "hashes": [ + "sha256:428266a1c0d36dc5aca63a2d7c5942e88c2c898d72139fca0e97fdd2380517ae", + "sha256:8ea2d3ce97850f31e4a08b0e2b5e6c34997d7216a9d2c98e0f3978630d4da69a" + ], + "version": "==3.1.1" + }, "certifi": { "hashes": [ "sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939", @@ -266,6 +273,13 @@ ], "version": "==4.8.0" }, + "cachetools": { + "hashes": [ + "sha256:428266a1c0d36dc5aca63a2d7c5942e88c2c898d72139fca0e97fdd2380517ae", + "sha256:8ea2d3ce97850f31e4a08b0e2b5e6c34997d7216a9d2c98e0f3978630d4da69a" + ], + "version": "==3.1.1" + }, "certifi": { "hashes": [ "sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939", diff --git a/pymisp/abstract.py b/pymisp/abstract.py index c85ef3b..ad273ed 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -4,9 +4,11 @@ import sys import datetime import json +import os from json import JSONEncoder import logging from enum import Enum +import cachetools from .exceptions import PyMISPInvalidFormat @@ -38,6 +40,12 @@ if sys.version_info < (3, 0): return timedelta(0) +if (3, 0) <= sys.version_info < (3, 6): + OLD_PY3 = True +else: + OLD_PY3 = False + + class Distribution(Enum): your_organisation_only = 0 this_community_only = 1 @@ -80,7 +88,30 @@ class MISPEncode(JSONEncoder): return JSONEncoder.default(self, obj) -class AbstractMISP(MutableMapping): +class MISPFileCache(object): + # cache up to 150 JSON structures in class attribute + _file_cache = cachetools.LFUCache(150) + + @staticmethod + def _load_json(path): + # use hard-coded root class attribute + file_cache = MISPFileCache._file_cache + # use modified time with path as cache key + mtime = os.path.getmtime(path) + if path in file_cache: + ctime, data = file_cache[path] + if ctime == mtime: + return data + with open(path, 'rb') as f: + if OLD_PY3: + data = json.loads(f.read().decode()) + else: + data = json.load(f) + file_cache[path] = (mtime, data) + return data + + +class AbstractMISP(MutableMapping, MISPFileCache): def __init__(self, **kwargs): """Abstract class for all the MISP objects""" diff --git a/pymisp/api.py b/pymisp/api.py index 6f2963e..4134312 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -19,7 +19,7 @@ from deprecated import deprecated from . import __version__, warning_2020 from .exceptions import PyMISPError, SearchError, NoURL, NoKey, PyMISPEmptyResponse from .mispevent import MISPEvent, MISPAttribute, MISPUser, MISPOrganisation, MISPSighting, MISPFeed, MISPObject, MISPSharingGroup -from .abstract import AbstractMISP, MISPEncode +from .abstract import AbstractMISP, MISPEncode, MISPFileCache logger = logging.getLogger('pymisp') @@ -37,11 +37,6 @@ try: except ImportError: HAVE_REQUESTS = False -if (3, 0) <= sys.version_info < (3, 6): - OLD_PY3 = True -else: - OLD_PY3 = False - try: from requests_futures.sessions import FuturesSession ASYNC_OK = True @@ -58,7 +53,7 @@ Response (if any): {}''' -class PyMISP(object): # pragma: no cover +class PyMISP(MISPFileCache): # pragma: no cover """Python API for MISP :param url: URL of the MISP instance you want to connect to @@ -140,11 +135,7 @@ class PyMISP(object): # pragma: no cover @deprecated(reason="Use ExpandedPyMISP.describe_types_local", version='2.4.110') def get_local_describe_types(self): - with open(os.path.join(self.resources_path, 'describeTypes.json'), 'rb') as f: - if OLD_PY3: - describe_types = json.loads(f.read().decode()) - else: - describe_types = json.load(f) + describe_types = self._load_json(os.path.join(self.resources_path, 'describeTypes.json')) return describe_types['result'] @deprecated(reason="Use ExpandedPyMISP.describe_types_remote", version='2.4.110') diff --git a/pymisp/aping.py b/pymisp/aping.py index 99b14d6..3885db3 100644 --- a/pymisp/aping.py +++ b/pymisp/aping.py @@ -105,8 +105,7 @@ class ExpandedPyMISP(PyMISP): @property def describe_types_local(self): '''Returns the content of describe types from the package''' - with (self.resources_path / 'describeTypes.json').open() as f: - describe_types = json.load(f) + describe_types = self._load_json(str(self.resources_path / 'describeTypes.json')) return describe_types['result'] @property diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 15a3550..e3507f8 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -109,11 +109,7 @@ class MISPAttribute(AbstractMISP): super(MISPAttribute, self).__init__() if not describe_types: ressources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') - with open(os.path.join(ressources_path, 'describeTypes.json'), 'rb') as f: - if OLD_PY3: - t = json.loads(f.read().decode()) - else: - t = json.load(f) + t = self._load_json(os.path.join(ressources_path, 'describeTypes.json')) describe_types = t['result'] self.__categories = describe_types['categories'] self._types = describe_types['types'] @@ -411,26 +407,14 @@ class MISPEvent(AbstractMISP): super(MISPEvent, self).__init__(**kwargs) ressources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') if strict_validation: - with open(os.path.join(ressources_path, 'schema.json'), 'rb') as f: - if OLD_PY3: - self.__json_schema = json.loads(f.read().decode()) - else: - self.__json_schema = json.load(f) + self.__json_schema = self._load_json(os.path.join(ressources_path, 'schema.json')) else: - with open(os.path.join(ressources_path, 'schema-lax.json'), 'rb') as f: - if OLD_PY3: - self.__json_schema = json.loads(f.read().decode()) - else: - self.__json_schema = json.load(f) + self.__json_schema = self._load_json(os.path.join(ressources_path, 'schema-lax.json')) if describe_types: # This variable is used in add_attribute in order to avoid duplicating the structure self._describe_types = describe_types else: - with open(os.path.join(ressources_path, 'describeTypes.json'), 'rb') as f: - if OLD_PY3: - t = json.loads(f.read().decode()) - else: - t = json.load(f) + t = self._load_json(os.path.join(ressources_path, 'describeTypes.json')) self._describe_types = t['result'] self._types = self._describe_types['types'] @@ -1190,11 +1174,7 @@ class MISPObject(AbstractMISP): def _load_template_path(self, template_path): if not os.path.exists(template_path): return False - with open(template_path, 'rb') as f: - if OLD_PY3: - self._definition = json.loads(f.read().decode()) - else: - self._definition = json.load(f) + self._definition = self._load_json(template_path) setattr(self, 'meta-category', self._definition['meta-category']) self.template_uuid = self._definition['uuid'] self.description = self._definition['description'] diff --git a/setup.py b/setup.py index 83055d9..be3284b 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ setup( ], install_requires=['six', 'requests', 'python-dateutil', 'jsonschema', 'python-dateutil', 'enum34;python_version<"3.4"', - 'functools32;python_version<"3.0"', 'deprecated'], + 'functools32;python_version<"3.0"', 'deprecated', 'cachetools'], extras_require={'fileobjects': ['lief>=0.8,<0.10;python_version<"3.5"', 'lief>=0.10.0.dev0;python_version>"3.5"', 'python-magic', 'pydeep'], 'neo': ['py2neo'], 'openioc': ['beautifulsoup4'], From 6c1f988b131ffa4f861983aa6693caf192e09950 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 3 Oct 2019 13:23:00 -0700 Subject: [PATCH 03/12] fix: Cache describeTypes at AbstractMISP level. --- pymisp/abstract.py | 31 +++++++++++++++++++++++++++++++ pymisp/mispevent.py | 21 +++++---------------- 2 files changed, 36 insertions(+), 16 deletions(-) diff --git a/pymisp/abstract.py b/pymisp/abstract.py index 39c6165..0657765 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -80,8 +80,35 @@ class MISPEncode(JSONEncoder): return JSONEncoder.default(self, obj) +if sys.version_info >= (3, 6): + from pathlib import Path + + def cache_describe_types(): + resources_path = Path(__file__).parent / 'data' + with (resources_path / 'describeTypes.json').open() as f: + dt = json.load(f) + return dt['result'] +else: + import os + if (3, 0) <= sys.version_info < (3, 6): + OLD_PY3 = True + else: + OLD_PY3 = False + + def cache_describe_types(): + ressources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') + with open(os.path.join(ressources_path, 'describeTypes.json'), 'rb') as f: + if OLD_PY3: + t = json.loads(f.read().decode()) + else: + t = json.load(f) + return t['result'] + + class AbstractMISP(MutableMapping): + __describe_types = cache_describe_types() + def __init__(self, **kwargs): """Abstract class for all the MISP objects""" super(AbstractMISP, self).__init__() @@ -102,6 +129,10 @@ class AbstractMISP(MutableMapping): setattr(AbstractMISP, 'add_tag', AbstractMISP.__add_tag) setattr(AbstractMISP, 'tags', property(AbstractMISP.__get_tags, AbstractMISP.__set_tags)) + @property + def describe_types(self): + return self.__describe_types + @property def properties(self): """All the class public properties that will be dumped in the dictionary, and the JSON export. diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index d5ef5ab..15af455 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -110,15 +110,9 @@ class MISPAttribute(AbstractMISP): """ super(MISPAttribute, self).__init__() if not describe_types: - ressources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') - with open(os.path.join(ressources_path, 'describeTypes.json'), 'rb') as f: - if OLD_PY3: - t = json.loads(f.read().decode()) - else: - t = json.load(f) - describe_types = t['result'] - self.__categories = describe_types['categories'] - self._types = describe_types['types'] + describe_types = self.describe_types + self.__categories = frozenset(describe_types['categories']) + self._types = frozenset(describe_types['types']) self.__category_type_mapping = describe_types['category_type_mappings'] self.__sane_default = describe_types['sane_defaults'] self.__strict = strict @@ -442,14 +436,9 @@ class MISPEvent(AbstractMISP): # This variable is used in add_attribute in order to avoid duplicating the structure self._describe_types = describe_types else: - with open(os.path.join(ressources_path, 'describeTypes.json'), 'rb') as f: - if OLD_PY3: - t = json.loads(f.read().decode()) - else: - t = json.load(f) - self._describe_types = t['result'] + self._describe_types = self.describe_types - self._types = self._describe_types['types'] + self._types = frozenset(self._describe_types['types']) self.Attribute = [] self.Object = [] self.RelatedEvent = [] From bae942d2ecf9839a37e248c002f43952a83a2a4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 3 Oct 2019 13:54:39 -0700 Subject: [PATCH 04/12] fix: Cache object templates at AbstractMISP level Related #468 and #471 --- pymisp/abstract.py | 20 ++++++++++++++++++++ pymisp/mispevent.py | 12 ++++-------- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/pymisp/abstract.py b/pymisp/abstract.py index 0657765..e49adba 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -88,6 +88,12 @@ if sys.version_info >= (3, 6): with (resources_path / 'describeTypes.json').open() as f: dt = json.load(f) return dt['result'] + + def load_template(path): + with open(path) as f: + t = json.load(f) + return t + else: import os if (3, 0) <= sys.version_info < (3, 6): @@ -104,10 +110,19 @@ else: t = json.load(f) return t['result'] + def load_template(path): + with open(path, 'rb') as f: + if OLD_PY3: + t = json.loads(f.read().decode()) + else: + t = json.load(f) + return t + class AbstractMISP(MutableMapping): __describe_types = cache_describe_types() + __object_templates = {} def __init__(self, **kwargs): """Abstract class for all the MISP objects""" @@ -133,6 +148,11 @@ class AbstractMISP(MutableMapping): def describe_types(self): return self.__describe_types + def get_template_definition(self, template_path): + if template_path not in self.__object_templates: + self.__object_templates[template_path] = load_template(template_path) + return self.__object_templates[template_path] + @property def properties(self): """All the class public properties that will be dumped in the dictionary, and the JSON export. diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 15af455..7939813 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -111,8 +111,8 @@ class MISPAttribute(AbstractMISP): super(MISPAttribute, self).__init__() if not describe_types: describe_types = self.describe_types - self.__categories = frozenset(describe_types['categories']) - self._types = frozenset(describe_types['types']) + self.__categories = describe_types['categories'] + self._types = describe_types['types'] self.__category_type_mapping = describe_types['category_type_mappings'] self.__sane_default = describe_types['sane_defaults'] self.__strict = strict @@ -438,7 +438,7 @@ class MISPEvent(AbstractMISP): else: self._describe_types = self.describe_types - self._types = frozenset(self._describe_types['types']) + self._types = self._describe_types['types'] self.Attribute = [] self.Object = [] self.RelatedEvent = [] @@ -1195,11 +1195,7 @@ class MISPObject(AbstractMISP): def _load_template_path(self, template_path): if not os.path.exists(template_path): return False - with open(template_path, 'rb') as f: - if OLD_PY3: - self._definition = json.loads(f.read().decode()) - else: - self._definition = json.load(f) + self._definition = self.get_template_definition(template_path) setattr(self, 'meta-category', self._definition['meta-category']) self.template_uuid = self._definition['uuid'] self.description = self._definition['description'] From 4be029a0f6fb25c8e485cb6dbc497e7b41636c1d Mon Sep 17 00:00:00 2001 From: Marc Hoersken Date: Fri, 4 Oct 2019 08:55:55 +0200 Subject: [PATCH 05/12] Use classmethod instead of staticmethod and avoid hard-coded reference --- pymisp/abstract.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pymisp/abstract.py b/pymisp/abstract.py index ad273ed..2fbc8c5 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -90,12 +90,12 @@ class MISPEncode(JSONEncoder): class MISPFileCache(object): # cache up to 150 JSON structures in class attribute - _file_cache = cachetools.LFUCache(150) + __file_cache = cachetools.LFUCache(150) - @staticmethod - def _load_json(path): - # use hard-coded root class attribute - file_cache = MISPFileCache._file_cache + @classmethod + def _load_json(cls, path): + # use root class attribute as global cache + file_cache = cls.__file_cache # use modified time with path as cache key mtime = os.path.getmtime(path) if path in file_cache: From f322e479cdf0d353c8356af767afc5906b50ffcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 7 Oct 2019 04:00:11 -0600 Subject: [PATCH 06/12] fix: Support for legacy python versions 90 days and counting, folks. --- pymisp/abstract.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymisp/abstract.py b/pymisp/abstract.py index 651292f..9ffa86d 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -68,7 +68,7 @@ else: resources_path = Path(__file__).parent / 'data' misp_objects_path = resources_path / 'misp-objects' / 'objects' - with (resources_path / 'describeTypes.json').open('rb') as f: + with (resources_path / 'describeTypes.json').open('r') as f: describe_types = json.load(f)['result'] class MISPFileCache(object): @@ -76,7 +76,7 @@ else: @classmethod @lru_cache(maxsize=150) - def _load_json(cls, path: Path): + def _load_json(cls, path): with path.open('rb') as f: data = json.load(f) return data From 3bb220c94badac84b61e99e4294abc02dd7c63b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 8 Oct 2019 08:15:56 +0200 Subject: [PATCH 07/12] chg: Cleanups and improvements --- pymisp/abstract.py | 6 +++-- pymisp/api.py | 5 ++--- pymisp/aping.py | 5 ++--- pymisp/mispevent.py | 54 +++++++++++++++++++-------------------------- 4 files changed, 31 insertions(+), 39 deletions(-) diff --git a/pymisp/abstract.py b/pymisp/abstract.py index 9ffa86d..6fc05d3 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -74,9 +74,9 @@ else: class MISPFileCache(object): # cache up to 150 JSON structures in class attribute - @classmethod + @staticmethod @lru_cache(maxsize=150) - def _load_json(cls, path): + def _load_json(path): with path.open('rb') as f: data = json.load(f) return data @@ -173,6 +173,8 @@ class AbstractMISP(MutableMapping, MISPFileCache): @misp_objects_path.setter def misp_objects_path(self, misp_objects_path): + if sys.version_info >= (3, 0) and isinstance(misp_objects_path, str): + misp_objects_path = Path(misp_objects_path) self.__misp_objects_path = misp_objects_path @property diff --git a/pymisp/api.py b/pymisp/api.py index 030262a..58da773 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -19,7 +19,7 @@ from deprecated import deprecated from . import __version__, warning_2020 from .exceptions import PyMISPError, SearchError, NoURL, NoKey, PyMISPEmptyResponse from .mispevent import MISPEvent, MISPAttribute, MISPUser, MISPOrganisation, MISPSighting, MISPFeed, MISPObject, MISPSharingGroup -from .abstract import AbstractMISP, MISPEncode, MISPFileCache +from .abstract import AbstractMISP, MISPEncode, MISPFileCache, describe_types logger = logging.getLogger('pymisp') @@ -135,8 +135,7 @@ class PyMISP(MISPFileCache): # pragma: no cover @deprecated(reason="Use ExpandedPyMISP.describe_types_local", version='2.4.110', action='default') def get_local_describe_types(self): - describe_types = self._load_json(os.path.join(self.resources_path, 'describeTypes.json')) - return describe_types['result'] + return describe_types @deprecated(reason="Use ExpandedPyMISP.describe_types_remote", version='2.4.110', action='default') def get_live_describe_types(self): diff --git a/pymisp/aping.py b/pymisp/aping.py index c9b3e32..931aca2 100644 --- a/pymisp/aping.py +++ b/pymisp/aping.py @@ -19,7 +19,7 @@ from . import __version__ from .exceptions import MISPServerError, PyMISPUnexpectedResponse, PyMISPNotImplementedYet, PyMISPError, NoURL, NoKey from .api import everything_broken, PyMISP from .mispevent import MISPEvent, MISPAttribute, MISPSighting, MISPLog, MISPObject, MISPUser, MISPOrganisation, MISPShadowAttribute, MISPWarninglist, MISPTaxonomy, MISPGalaxy, MISPNoticelist, MISPObjectReference, MISPObjectTemplate, MISPSharingGroup, MISPRole, MISPServer, MISPFeed, MISPEventDelegation, MISPCommunity -from .abstract import MISPEncode, MISPTag, AbstractMISP +from .abstract import MISPEncode, MISPTag, AbstractMISP, describe_types SearchType = TypeVar('SearchType', str, int) # str: string to search / list: values to search (OR) / dict: {'OR': [list], 'NOT': [list], 'AND': [list]} @@ -106,8 +106,7 @@ class ExpandedPyMISP(PyMISP): @property def describe_types_local(self): '''Returns the content of describe types from the package''' - describe_types = self._load_json(str(self.resources_path / 'describeTypes.json')) - return describe_types['result'] + return describe_types @property def describe_types_remote(self): diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 2a05553..e804908 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -113,7 +113,7 @@ class MISPAttribute(AbstractMISP): if describe_types: self.describe_types = describe_types self.__categories = self.describe_types['categories'] - self._types = self.describe_types['types'] + self.__types = self.describe_types['types'] self.__category_type_mapping = self.describe_types['category_type_mappings'] self.__sane_default = self.describe_types['sane_defaults'] self.__strict = strict @@ -125,7 +125,7 @@ class MISPAttribute(AbstractMISP): @property def known_types(self): """Returns a list of all the known MISP attributes types""" - return self._types + return self.__types @property def malware_binary(self): @@ -213,7 +213,7 @@ class MISPAttribute(AbstractMISP): if self.type is None: raise NewAttributeError('The type of the attribute is required.') if self.type not in self.known_types: - raise NewAttributeError('{} is invalid, type has to be in {}'.format(self.type, (', '.join(self._types)))) + raise NewAttributeError('{} is invalid, type has to be in {}'.format(self.type, (', '.join(self.known_types)))) type_defaults = self.__sane_default[self.type] @@ -434,7 +434,7 @@ class MISPEvent(AbstractMISP): # This variable is used in add_attribute in order to avoid duplicating the structure self.describe_types = describe_types - self._types = self.describe_types['types'] + self.__types = self.describe_types['types'] self.Attribute = [] self.Object = [] self.RelatedEvent = [] @@ -442,7 +442,7 @@ class MISPEvent(AbstractMISP): @property def known_types(self): - return self._types + return self.__types @property def org(self): @@ -1140,25 +1140,7 @@ class MISPObject(AbstractMISP): self.name = name self._known_template = False - if kwargs.get('misp_objects_path_custom'): - # If misp_objects_path_custom is given, and an object with the given name exists, use that. - if sys.version_info >= (3, 6): - self._known_template = self._load_template_path(Path(kwargs.get('misp_objects_path_custom')) / self.name / 'definition.json') - else: - self._known_template = self._load_template_path(os.path.join(kwargs.get('misp_objects_path_custom'), self.name, 'definition.json')) - - if not self._known_template: - # Check if the object is known in the default templates bundled in with PyMISP - if sys.version_info >= (3, 6): - self._known_template = self._load_template_path(self.misp_objects_path / self.name / 'definition.json') - else: - self._known_template = self._load_template_path(os.path.join(self.misp_objects_path, self.name, 'definition.json')) - - if not self._known_template and self._strict: - raise UnknownMISPObjectTemplate('{} is unknown in the MISP object directory.'.format(self.name)) - else: - # Then we have no meta-category, template_uuid, description and template_version - pass + self._set_template(kwargs.get('misp_objects_path_custom')) self.uuid = str(uuid.uuid4()) self.__fast_attribute_access = defaultdict(list) # Hashtable object_relation: [attributes] @@ -1206,14 +1188,24 @@ class MISPObject(AbstractMISP): def force_misp_objects_path_custom(self, misp_objects_path_custom, object_name=None): if object_name: self.name = object_name - if sys.version_info >= (3, 6): - template_path = Path(misp_objects_path_custom) / self.name / 'definition.json' - else: - template_path = os.path.join(misp_objects_path_custom, self.name, 'definition.json') + self._set_template(misp_objects_path_custom) - self._known_template = self._load_template_path(template_path) - if not self._known_template: - raise UnknownMISPObjectTemplate('{} is unknown in the MISP object directory ({}).'.format(self.name, template_path)) + def _set_template(self, misp_objects_path_custom=None): + if misp_objects_path_custom: + # If misp_objects_path_custom is given, and an object with the given name exists, use that. + self.misp_objects_path = misp_objects_path_custom + + # Try to get the template + if sys.version_info >= (3, 6): + self._known_template = self._load_template_path(self.misp_objects_path / self.name / 'definition.json') + else: + self._known_template = self._load_template_path(os.path.join(self.misp_objects_path, self.name, 'definition.json')) + + if not self._known_template and self._strict: + raise UnknownMISPObjectTemplate('{} is unknown in the MISP object directory.'.format(self.name)) + else: + # Then we have no meta-category, template_uuid, description and template_version + pass @property def disable_validation(self): From f312f870722ffb655040d3572365b7c247def392 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 8 Oct 2019 09:28:33 +0200 Subject: [PATCH 08/12] fix: Objects helpers were broken, do not overwrite describe_types --- pymisp/abstract.py | 6 ++++-- pymisp/api.py | 12 ++++++------ pymisp/aping.py | 4 ++-- pymisp/tools/elfobject.py | 2 +- pymisp/tools/emailobject.py | 6 +++--- pymisp/tools/fileobject.py | 6 +++--- pymisp/tools/machoobject.py | 6 +++--- pymisp/tools/peobject.py | 6 +++--- pymisp/tools/sshauthkeyobject.py | 6 +++--- tests/testlive_comprehensive.py | 2 +- 10 files changed, 29 insertions(+), 27 deletions(-) diff --git a/pymisp/abstract.py b/pymisp/abstract.py index 6fc05d3..fa1a015 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -134,12 +134,12 @@ class AbstractMISP(MutableMapping, MISPFileCache): __misp_objects_path = misp_objects_path __describe_types = describe_types - def __init__(self, **kwargs): """Abstract class for all the MISP objects""" super(AbstractMISP, self).__init__() self.__edited = True # As we create a new object, we assume it is edited self.__not_jsonable = [] + self.__self_defined_describe_types = None if kwargs.get('force_timestamps') is not None: # Ignore the edited objects and keep the timestamps. @@ -157,11 +157,13 @@ class AbstractMISP(MutableMapping, MISPFileCache): @property def describe_types(self): + if self.__self_defined_describe_types: + return self.__self_defined_describe_types return self.__describe_types @describe_types.setter def describe_types(self, describe_types): - self.__describe_types = describe_types + self.__self_defined_describe_types = describe_types @property def resources_path(self): diff --git a/pymisp/api.py b/pymisp/api.py index 58da773..bc7a4af 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -140,14 +140,14 @@ class PyMISP(MISPFileCache): # pragma: no cover @deprecated(reason="Use ExpandedPyMISP.describe_types_remote", version='2.4.110', action='default') def get_live_describe_types(self): response = self._prepare_request('GET', urljoin(self.root_url, 'attributes/describeTypes.json')) - describe_types = self._check_response(response) - if describe_types.get('error'): - for e in describe_types.get('error'): + remote_describe_types = self._check_response(response) + if remote_describe_types.get('error'): + for e in remote_describe_types.get('error'): raise PyMISPError('Failed: {}'.format(e)) - describe_types = describe_types['result'] - if not describe_types.get('sane_defaults'): + remote_describe_types = describe_types['result'] + if not remote_describe_types.get('sane_defaults'): raise PyMISPError('The MISP server your are trying to reach is outdated (<2.4.52). Please use PyMISP v2.4.51.1 (pip install -I PyMISP==v2.4.51.1) and/or contact your administrator.') - return describe_types + return remote_describe_types def _prepare_request(self, request_type, url, data=None, background_callback=None, output_type='json'): diff --git a/pymisp/aping.py b/pymisp/aping.py index 931aca2..2a2a24a 100644 --- a/pymisp/aping.py +++ b/pymisp/aping.py @@ -112,8 +112,8 @@ class ExpandedPyMISP(PyMISP): def describe_types_remote(self): '''Returns the content of describe types from the remote instance''' response = self._prepare_request('GET', 'attributes/describeTypes.json') - describe_types = self._check_response(response, expect_json=True) - return describe_types['result'] + remote_describe_types = self._check_response(response, expect_json=True) + return remote_describe_types['result'] @property def recommended_pymisp_version(self): diff --git a/pymisp/tools/elfobject.py b/pymisp/tools/elfobject.py index 2dcdeb0..e57654b 100644 --- a/pymisp/tools/elfobject.py +++ b/pymisp/tools/elfobject.py @@ -25,6 +25,7 @@ except ImportError: class ELFObject(AbstractMISPObjectGenerator): def __init__(self, parsed=None, filepath=None, pseudofile=None, standalone=True, **kwargs): + super(ELFObject, self).__init__('elf', standalone=standalone, **kwargs) if not HAS_PYDEEP: logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") if not HAS_LIEF: @@ -44,7 +45,6 @@ class ELFObject(AbstractMISPObjectGenerator): self.__elf = parsed else: raise InvalidMISPObject('Not a lief.ELF.Binary: {}'.format(type(parsed))) - super(ELFObject, self).__init__('elf', standalone=standalone, **kwargs) self.generate_attributes() def generate_attributes(self): diff --git a/pymisp/tools/emailobject.py b/pymisp/tools/emailobject.py index 4a5b54f..0984336 100644 --- a/pymisp/tools/emailobject.py +++ b/pymisp/tools/emailobject.py @@ -13,6 +13,9 @@ logger = logging.getLogger('pymisp') class EMailObject(AbstractMISPObjectGenerator): def __init__(self, filepath=None, pseudofile=None, attach_original_email=True, standalone=True, **kwargs): + # PY3 way: + # super().__init__('file') + super(EMailObject, self).__init__('email', standalone=standalone, **kwargs) if filepath: with open(filepath, 'rb') as f: self.__pseudofile = BytesIO(f.read()) @@ -20,9 +23,6 @@ class EMailObject(AbstractMISPObjectGenerator): self.__pseudofile = pseudofile else: raise InvalidMISPObject('File buffer (BytesIO) or a path is required.') - # PY3 way: - # super().__init__('file') - super(EMailObject, self).__init__('email', standalone=standalone, **kwargs) self.__email = message_from_bytes(self.__pseudofile.getvalue(), policy=policy.default) if attach_original_email: self.add_attribute('eml', value='Full email.eml', data=self.__pseudofile) diff --git a/pymisp/tools/fileobject.py b/pymisp/tools/fileobject.py index e9b05cd..55e946c 100644 --- a/pymisp/tools/fileobject.py +++ b/pymisp/tools/fileobject.py @@ -29,6 +29,9 @@ except ImportError: class FileObject(AbstractMISPObjectGenerator): def __init__(self, filepath=None, pseudofile=None, filename=None, standalone=True, **kwargs): + # PY3 way: + # super().__init__('file') + super(FileObject, self).__init__('file', standalone=standalone, **kwargs) if not HAS_PYDEEP: logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") if not HAS_MAGIC: @@ -49,9 +52,6 @@ class FileObject(AbstractMISPObjectGenerator): self.__pseudofile = pseudofile else: raise InvalidMISPObject('File buffer (BytesIO) or a path is required.') - # PY3 way: - # super().__init__('file') - super(FileObject, self).__init__('file', standalone=standalone, **kwargs) self.__data = self.__pseudofile.getvalue() self.generate_attributes() diff --git a/pymisp/tools/machoobject.py b/pymisp/tools/machoobject.py index f0f11fb..984798b 100644 --- a/pymisp/tools/machoobject.py +++ b/pymisp/tools/machoobject.py @@ -26,6 +26,9 @@ except ImportError: class MachOObject(AbstractMISPObjectGenerator): def __init__(self, parsed=None, filepath=None, pseudofile=None, standalone=True, **kwargs): + # Python3 way + # super().__init__('elf') + super(MachOObject, self).__init__('macho', standalone=standalone, **kwargs) if not HAS_PYDEEP: logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") if not HAS_LIEF: @@ -45,9 +48,6 @@ class MachOObject(AbstractMISPObjectGenerator): self.__macho = parsed else: raise InvalidMISPObject('Not a lief.MachO.Binary: {}'.format(type(parsed))) - # Python3 way - # super().__init__('elf') - super(MachOObject, self).__init__('macho', standalone=standalone, **kwargs) self.generate_attributes() def generate_attributes(self): diff --git a/pymisp/tools/peobject.py b/pymisp/tools/peobject.py index b24d4cc..fa1b4e5 100644 --- a/pymisp/tools/peobject.py +++ b/pymisp/tools/peobject.py @@ -26,6 +26,9 @@ except ImportError: class PEObject(AbstractMISPObjectGenerator): def __init__(self, parsed=None, filepath=None, pseudofile=None, standalone=True, **kwargs): + # Python3 way + # super().__init__('pe') + super(PEObject, self).__init__('pe', standalone=standalone, **kwargs) if not HAS_PYDEEP: logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") if not HAS_LIEF: @@ -45,9 +48,6 @@ class PEObject(AbstractMISPObjectGenerator): self.__pe = parsed else: raise InvalidMISPObject('Not a lief.PE.Binary: {}'.format(type(parsed))) - # Python3 way - # super().__init__('pe') - super(PEObject, self).__init__('pe', standalone=standalone, **kwargs) self.generate_attributes() def _is_exe(self): diff --git a/pymisp/tools/sshauthkeyobject.py b/pymisp/tools/sshauthkeyobject.py index 30f675d..95bb937 100644 --- a/pymisp/tools/sshauthkeyobject.py +++ b/pymisp/tools/sshauthkeyobject.py @@ -12,6 +12,9 @@ logger = logging.getLogger('pymisp') class SSHAuthorizedKeysObject(AbstractMISPObjectGenerator): def __init__(self, authorized_keys_path=None, authorized_keys_pseudofile=None, standalone=True, **kwargs): + # PY3 way: + # super().__init__('file') + super(SSHAuthorizedKeysObject, self).__init__('ssh-authorized-keys', standalone=standalone, **kwargs) if authorized_keys_path: with open(authorized_keys_path, 'r') as f: self.__pseudofile = StringIO(f.read()) @@ -19,9 +22,6 @@ class SSHAuthorizedKeysObject(AbstractMISPObjectGenerator): self.__pseudofile = authorized_keys_path else: raise InvalidMISPObject('File buffer (StringIO) or a path is required.') - # PY3 way: - # super().__init__('file') - super(SSHAuthorizedKeysObject, self).__init__('ssh-authorized-keys', standalone=standalone, **kwargs) self.__data = self.__pseudofile.getvalue() self.generate_attributes() diff --git a/tests/testlive_comprehensive.py b/tests/testlive_comprehensive.py index 628de3f..170c6eb 100644 --- a/tests/testlive_comprehensive.py +++ b/tests/testlive_comprehensive.py @@ -1577,7 +1577,7 @@ class TestComprehensive(unittest.TestCase): remote_types = remote.pop('types') remote_categories = remote.pop('categories') remote_category_type_mappings = remote.pop('category_type_mappings') - local = self.admin_misp_connector.describe_types_local + local = dict(self.admin_misp_connector.describe_types_local) local_types = local.pop('types') local_categories = local.pop('categories') local_category_type_mappings = local.pop('category_type_mappings') From 898bc96ea6b317e9c9bdc4a62acf50ce16d6e3ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 8 Oct 2019 16:06:28 +0200 Subject: [PATCH 09/12] chg: Cleanups --- pymisp/mispevent.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index e804908..b81620c 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -113,7 +113,6 @@ class MISPAttribute(AbstractMISP): if describe_types: self.describe_types = describe_types self.__categories = self.describe_types['categories'] - self.__types = self.describe_types['types'] self.__category_type_mapping = self.describe_types['category_type_mappings'] self.__sane_default = self.describe_types['sane_defaults'] self.__strict = strict @@ -125,7 +124,7 @@ class MISPAttribute(AbstractMISP): @property def known_types(self): """Returns a list of all the known MISP attributes types""" - return self.__types + return self.describe_types['types'] @property def malware_binary(self): @@ -421,20 +420,17 @@ class MISPEvent(AbstractMISP): def __init__(self, describe_types=None, strict_validation=False, **kwargs): super(MISPEvent, self).__init__(**kwargs) if strict_validation: - if sys.version_info >= (3, 6): - self.__json_schema = self._load_json(self.resources_path / 'schema.json') - else: - self.__json_schema = self._load_json(os.path.join(self.resources_path, 'schema.json')) + schema_file = 'schema.json' else: - if sys.version_info >= (3, 6): - self.__json_schema = self._load_json(self.resources_path / 'schema-lax.json') - else: - self.__json_schema = self._load_json(os.path.join(self.resources_path, 'schema-lax.json')) + schema_file = 'schema-lax.json' + if sys.version_info >= (3, 6): + self.__json_schema = self._load_json(self.resources_path / schema_file) + else: + self.__json_schema = self._load_json(os.path.join(self.resources_path, schema_file)) if describe_types: # This variable is used in add_attribute in order to avoid duplicating the structure self.describe_types = describe_types - self.__types = self.describe_types['types'] self.Attribute = [] self.Object = [] self.RelatedEvent = [] @@ -442,7 +438,7 @@ class MISPEvent(AbstractMISP): @property def known_types(self): - return self.__types + return self.describe_types['types'] @property def org(self): From 02659a5782ded0234337ab97b542316554cdc6c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 9 Oct 2019 16:07:40 +0200 Subject: [PATCH 10/12] chg: Add support for rapidjson, refactoring and code cleanup. --- examples/generate_file_objects.py | 5 +- pymisp/__init__.py | 2 +- pymisp/abstract.py | 93 +++++++++----- pymisp/api.py | 8 +- pymisp/aping.py | 4 +- pymisp/mispevent.py | 116 +++++++++--------- tests/mispevent_testfiles/existing_event.json | 4 +- .../existing_event_edited.json | 4 +- tests/test_mispevent.py | 41 ++++--- 9 files changed, 154 insertions(+), 123 deletions(-) diff --git a/examples/generate_file_objects.py b/examples/generate_file_objects.py index c3eda36..c187b9d 100755 --- a/examples/generate_file_objects.py +++ b/examples/generate_file_objects.py @@ -5,7 +5,7 @@ import argparse import json try: - from pymisp import MISPEncode, AbstractMISP + from pymisp import pymisp_json_default, AbstractMISP from pymisp.tools import make_binary_objects except ImportError: pass @@ -51,7 +51,8 @@ def make_objects(path): to_return['objects'].append(fo) if fo.ObjectReference: to_return['references'] += fo.ObjectReference - return json.dumps(to_return, cls=MISPEncode) + return json.dumps(to_return, default=pymisp_json_default) + if __name__ == '__main__': parser = argparse.ArgumentParser(description='Extract indicators out of binaries and returns MISP objects.') diff --git a/pymisp/__init__.py b/pymisp/__init__.py index f302be8..f134cce 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -31,7 +31,7 @@ try: warning_2020() from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate, PyMISPInvalidFormat, MISPServerError, PyMISPNotImplementedYet, PyMISPUnexpectedResponse, PyMISPEmptyResponse # noqa from .api import PyMISP # noqa - from .abstract import AbstractMISP, MISPEncode, MISPTag, Distribution, ThreatLevel, Analysis # noqa + from .abstract import AbstractMISP, MISPEncode, pymisp_json_default, MISPTag, Distribution, ThreatLevel, Analysis # noqa from .mispevent import MISPEvent, MISPAttribute, MISPObjectReference, MISPObjectAttribute, MISPObject, MISPUser, MISPOrganisation, MISPSighting, MISPLog, MISPShadowAttribute, MISPWarninglist, MISPTaxonomy, MISPNoticelist, MISPObjectTemplate, MISPSharingGroup, MISPRole, MISPServer, MISPFeed, MISPEventDelegation # noqa from .tools import AbstractMISPObjectGenerator # noqa from .tools import Neo4j # noqa diff --git a/pymisp/abstract.py b/pymisp/abstract.py index fa1a015..6921d53 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -3,8 +3,23 @@ import sys import datetime -import json + +from deprecated import deprecated from json import JSONEncoder + +try: + from rapidjson import load + from rapidjson import loads + from rapidjson import dumps + import rapidjson + HAS_RAPIDJSON = True +except ImportError: + from json import load + from json import loads + from json import dumps + import json + HAS_RAPIDJSON = False + import logging from enum import Enum @@ -13,7 +28,6 @@ from .exceptions import PyMISPInvalidFormat logger = logging.getLogger('pymisp') - if sys.version_info < (3, 0): from collections import MutableMapping import os @@ -22,7 +36,7 @@ if sys.version_info < (3, 0): resources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') misp_objects_path = os.path.join(resources_path, 'misp-objects', 'objects') with open(os.path.join(resources_path, 'describeTypes.json'), 'r') as f: - describe_types = json.load(f)['result'] + describe_types = load(f)['result'] # This is required because Python 2 is a pain. from datetime import tzinfo, timedelta @@ -55,9 +69,9 @@ if sys.version_info < (3, 0): return data with open(path, 'rb') as f: if OLD_PY3: - data = json.loads(f.read().decode()) + data = loads(f.read().decode()) else: - data = json.load(f) + data = load(f) file_cache[path] = (mtime, data) return data @@ -69,7 +83,7 @@ else: resources_path = Path(__file__).parent / 'data' misp_objects_path = resources_path / 'misp-objects' / 'objects' with (resources_path / 'describeTypes.json').open('r') as f: - describe_types = json.load(f)['result'] + describe_types = load(f)['result'] class MISPFileCache(object): # cache up to 150 JSON structures in class attribute @@ -78,7 +92,7 @@ else: @lru_cache(maxsize=150) def _load_json(path): with path.open('rb') as f: - data = json.load(f) + data = load(f) return data if (3, 0) <= sys.version_info < (3, 6): @@ -117,8 +131,8 @@ def _int_to_str(d): return d +@deprecated(reason=" Use method default=pymisp_json_default instead of cls=MISPEncode", version='2.4.117', action='default') class MISPEncode(JSONEncoder): - def default(self, obj): if isinstance(obj, AbstractMISP): return obj.jsonable() @@ -129,6 +143,26 @@ class MISPEncode(JSONEncoder): return JSONEncoder.default(self, obj) +if HAS_RAPIDJSON: + def pymisp_json_default(obj): + if isinstance(obj, AbstractMISP): + return obj.jsonable() + elif isinstance(obj, (datetime.datetime, datetime.date)): + return obj.isoformat() + elif isinstance(obj, Enum): + return obj.value + return rapidjson.default(obj) +else: + def pymisp_json_default(obj): + if isinstance(obj, AbstractMISP): + return obj.jsonable() + elif isinstance(obj, (datetime.datetime, datetime.date)): + return obj.isoformat() + elif isinstance(obj, Enum): + return obj.value + return json.default(obj) + + class AbstractMISP(MutableMapping, MISPFileCache): __resources_path = resources_path __misp_objects_path = misp_objects_path @@ -179,13 +213,6 @@ class AbstractMISP(MutableMapping, MISPFileCache): misp_objects_path = Path(misp_objects_path) self.__misp_objects_path = misp_objects_path - @property - def properties(self): - """All the class public properties that will be dumped in the dictionary, and the JSON export. - Note: all the properties starting with a `_` (private), or listed in __not_jsonable will be skipped. - """ - return [k for k in vars(self).keys() if not (k[0] == '_' or k in self.__not_jsonable)] - def from_dict(self, **kwargs): """Loading all the parameters as class properties, if they aren't `None`. This method aims to be called when all the properties requiring a special @@ -209,21 +236,21 @@ class AbstractMISP(MutableMapping, MISPFileCache): def from_json(self, json_string): """Load a JSON string""" - self.from_dict(**json.loads(json_string)) + self.from_dict(**loads(json_string)) def to_dict(self): - """Dump the lass to a dictionary. + """Dump the class to a dictionary. This method automatically removes the timestamp recursively in every object that has been edited is order to let MISP update the event accordingly.""" + is_edited = self.edited to_return = {} - for attribute in self.properties: - val = getattr(self, attribute, None) + for attribute, val in self.items(): if val is None: continue elif isinstance(val, list) and len(val) == 0: continue if attribute == 'timestamp': - if not self.__force_timestamps and self.edited: + if not self.__force_timestamps and is_edited: # In order to be accepted by MISP, the timestamp of an object # needs to be either newer, or None. # If the current object is marked as edited, the easiest is to @@ -239,13 +266,15 @@ class AbstractMISP(MutableMapping, MISPFileCache): """This method is used by the JSON encoder""" return self.to_dict() - def to_json(self): + def to_json(self, sort_keys=False, indent=None): """Dump recursively any class of type MISPAbstract to a json string""" - return json.dumps(self, cls=MISPEncode, sort_keys=True, indent=2) + return dumps(self, default=pymisp_json_default, sort_keys=sort_keys, indent=indent) def __getitem__(self, key): try: - return getattr(self, key) + if key[0] != '_' and key not in self.__not_jsonable: + return self.__dict__[key] + raise KeyError except AttributeError: # Expected by pop and other dict-related methods raise KeyError @@ -257,10 +286,10 @@ class AbstractMISP(MutableMapping, MISPFileCache): delattr(self, key) def __iter__(self): - return iter(self.to_dict()) + return iter({k: v for k, v in self.__dict__.items() if not (k[0] == '_' or k in self.__not_jsonable)}) def __len__(self): - return len(self.to_dict()) + return len([k for k in self.__dict__.keys() if not (k[0] == '_' or k in self.__not_jsonable)]) @property def edited(self): @@ -268,15 +297,14 @@ class AbstractMISP(MutableMapping, MISPFileCache): to the parent objects""" if self.__edited: return self.__edited - for p in self.properties: - if self.__edited: - break - val = getattr(self, p) + for p, val in self.items(): if isinstance(val, AbstractMISP) and val.edited: self.__edited = True + break elif isinstance(val, list) and all(isinstance(a, AbstractMISP) for a in val): if any(a.edited for a in val): self.__edited = True + break return self.__edited @edited.setter @@ -288,9 +316,10 @@ class AbstractMISP(MutableMapping, MISPFileCache): raise Exception('edited can only be True or False') def __setattr__(self, name, value): - if name != '_AbstractMISP__edited': - if not self.__edited and name in self.properties: - self.__edited = True + if name[0] != '_' and not self.__edited and name in self.keys(): + # The private members don't matter + # If we already have a key with that name, we're modifying it. + self.__edited = True super(AbstractMISP, self).__setattr__(name, value) def _datetime_to_timestamp(self, d): diff --git a/pymisp/api.py b/pymisp/api.py index bc7a4af..186cfa3 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -19,7 +19,7 @@ from deprecated import deprecated from . import __version__, warning_2020 from .exceptions import PyMISPError, SearchError, NoURL, NoKey, PyMISPEmptyResponse from .mispevent import MISPEvent, MISPAttribute, MISPUser, MISPOrganisation, MISPSighting, MISPFeed, MISPObject, MISPSharingGroup -from .abstract import AbstractMISP, MISPEncode, MISPFileCache, describe_types +from .abstract import AbstractMISP, pymisp_json_default, MISPFileCache, describe_types logger = logging.getLogger('pymisp') @@ -162,7 +162,7 @@ class PyMISP(MISPFileCache): # pragma: no cover if isinstance(data, dict): # Remove None values. data = {k: v for k, v in data.items() if v is not None} - data = json.dumps(data, cls=MISPEncode) + data = json.dumps(data, default=pymisp_json_default) req = requests.Request(request_type, url, data=data) if self.asynch and background_callback is not None: local_session = FuturesSession @@ -604,7 +604,7 @@ class PyMISP(MISPFileCache): # pragma: no cover else: data = attributes.to_json() # _prepare_request(...) returns a requests.Response Object - resp = self._prepare_request('POST', url, json.dumps(data, cls=MISPEncode)) + resp = self._prepare_request('POST', url, json.dumps(data, default=pymisp_json_default)) try: responses.append(resp.json()) except Exception: @@ -1058,7 +1058,7 @@ class PyMISP(MISPFileCache): # pragma: no cover url = urljoin(self.root_url, 'shadow_attributes/{}/{}'.format(path, id)) if path in ['add', 'edit']: query = {'request': {'ShadowAttribute': attribute}} - response = self._prepare_request('POST', url, json.dumps(query, cls=MISPEncode)) + response = self._prepare_request('POST', url, json.dumps(query, default=pymisp_json_default)) elif path == 'view': response = self._prepare_request('GET', url) else: # accept or discard diff --git a/pymisp/aping.py b/pymisp/aping.py index 2a2a24a..152612a 100644 --- a/pymisp/aping.py +++ b/pymisp/aping.py @@ -19,7 +19,7 @@ from . import __version__ from .exceptions import MISPServerError, PyMISPUnexpectedResponse, PyMISPNotImplementedYet, PyMISPError, NoURL, NoKey from .api import everything_broken, PyMISP from .mispevent import MISPEvent, MISPAttribute, MISPSighting, MISPLog, MISPObject, MISPUser, MISPOrganisation, MISPShadowAttribute, MISPWarninglist, MISPTaxonomy, MISPGalaxy, MISPNoticelist, MISPObjectReference, MISPObjectTemplate, MISPSharingGroup, MISPRole, MISPServer, MISPFeed, MISPEventDelegation, MISPCommunity -from .abstract import MISPEncode, MISPTag, AbstractMISP, describe_types +from .abstract import pymisp_json_default, MISPTag, AbstractMISP, describe_types SearchType = TypeVar('SearchType', str, int) # str: string to search / list: values to search (OR) / dict: {'OR': [list], 'NOT': [list], 'AND': [list]} @@ -2109,7 +2109,7 @@ class ExpandedPyMISP(PyMISP): if isinstance(data, dict): # Else, we can directly json encode. # Remove None values. data = {k: v for k, v in data.items() if v is not None} - data = json.dumps(data, cls=MISPEncode) + data = json.dumps(data, default=pymisp_json_default) if kw_params: # CakePHP params in URL diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index b81620c..574466f 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -198,8 +198,8 @@ class MISPAttribute(AbstractMISP): return misp_sighting def from_dict(self, **kwargs): - if kwargs.get('Attribute'): - kwargs = kwargs.get('Attribute') + if 'Attribute' in kwargs: + kwargs = kwargs['Attribute'] if kwargs.get('type') and kwargs.get('category'): if kwargs['type'] not in self.__category_type_mapping[kwargs['category']]: if self.__strict: @@ -272,14 +272,11 @@ class MISPAttribute(AbstractMISP): raise NewAttributeError('If the distribution is set to sharing group, a sharing group ID is required (cannot be {}).'.format(self.sharing_group_id)) if kwargs.get('Tag'): - for tag in kwargs.pop('Tag'): - self.add_tag(tag) + [self.add_tag(tag) for tag in kwargs.pop('Tag')] if kwargs.get('Sighting'): - for sighting in kwargs.pop('Sighting'): - self.add_sighting(sighting) + [self.add_sighting(sighting) for sighting in kwargs.pop('Sighting')] if kwargs.get('ShadowAttribute'): - for s_attr in kwargs.pop('ShadowAttribute'): - self.add_shadow_attribute(s_attr) + [self.add_shadow_attribute(s_attr) for s_attr in kwargs.pop('ShadowAttribute')] # If the user wants to disable correlation, let them. Defaults to False. self.disable_correlation = kwargs.pop("disable_correlation", False) @@ -535,8 +532,8 @@ class MISPEvent(AbstractMISP): raise NewEventError('Invalid format for the date: {} - {}'.format(date, type(date))) def from_dict(self, **kwargs): - if kwargs.get('Event'): - kwargs = kwargs.get('Event') + if 'Event' in kwargs: + kwargs = kwargs['Event'] # Required value self.info = kwargs.pop('info', None) if self.info is None: @@ -568,8 +565,7 @@ class MISPEvent(AbstractMISP): if kwargs.get('date'): self.set_date(kwargs.pop('date')) if kwargs.get('Attribute'): - for a in kwargs.pop('Attribute'): - self.add_attribute(**a) + [self.add_attribute(**a) for a in kwargs.pop('Attribute')] # All other keys if kwargs.get('id'): @@ -596,11 +592,9 @@ class MISPEvent(AbstractMISP): sub_event.load(rel_event) self.RelatedEvent.append({'Event': sub_event}) if kwargs.get('Tag'): - for tag in kwargs.pop('Tag'): - self.add_tag(tag) + [self.add_tag(tag) for tag in kwargs.pop('Tag')] if kwargs.get('Object'): - for obj in kwargs.pop('Object'): - self.add_object(obj) + [self.add_object(obj) for obj in kwargs.pop('Object')] if kwargs.get('Org'): self.Org = MISPOrganisation() self.Org.from_dict(**kwargs.pop('Org')) @@ -860,8 +854,8 @@ class MISPObjectReference(AbstractMISP): super(MISPObjectReference, self).__init__() def from_dict(self, **kwargs): - if kwargs.get('ObjectReference'): - kwargs = kwargs.get('ObjectReference') + if 'ObjectReference' in kwargs: + kwargs = kwargs['ObjectReference'] super(MISPObjectReference, self).from_dict(**kwargs) def __repr__(self): @@ -876,8 +870,8 @@ class MISPObjectTemplate(AbstractMISP): super(MISPObjectTemplate, self).__init__() def from_dict(self, **kwargs): - if kwargs.get('ObjectTemplate'): - kwargs = kwargs.get('ObjectTemplate') + if 'ObjectTemplate' in kwargs: + kwargs = kwargs['ObjectTemplate'] super(MISPObjectTemplate, self).from_dict(**kwargs) @@ -887,8 +881,8 @@ class MISPUser(AbstractMISP): super(MISPUser, self).__init__() def from_dict(self, **kwargs): - if kwargs.get('User'): - kwargs = kwargs.get('User') + if 'User' in kwargs: + kwargs = kwargs['User'] super(MISPUser, self).from_dict(**kwargs) def __repr__(self): @@ -903,8 +897,8 @@ class MISPOrganisation(AbstractMISP): super(MISPOrganisation, self).__init__() def from_dict(self, **kwargs): - if kwargs.get('Organisation'): - kwargs = kwargs.get('Organisation') + if 'Organisation' in kwargs: + kwargs = kwargs['Organisation'] super(MISPOrganisation, self).from_dict(**kwargs) @@ -914,8 +908,8 @@ class MISPFeed(AbstractMISP): super(MISPFeed, self).__init__() def from_dict(self, **kwargs): - if kwargs.get('Feed'): - kwargs = kwargs.get('Feed') + if 'Feed' in kwargs: + kwargs = kwargs['Feed'] super(MISPFeed, self).from_dict(**kwargs) @@ -925,8 +919,8 @@ class MISPWarninglist(AbstractMISP): super(MISPWarninglist, self).__init__() def from_dict(self, **kwargs): - if kwargs.get('Warninglist'): - kwargs = kwargs.get('Warninglist') + if 'Warninglist' in kwargs: + kwargs = kwargs['Warninglist'] super(MISPWarninglist, self).from_dict(**kwargs) @@ -936,8 +930,8 @@ class MISPTaxonomy(AbstractMISP): super(MISPTaxonomy, self).__init__() def from_dict(self, **kwargs): - if kwargs.get('Taxonomy'): - kwargs = kwargs.get('Taxonomy') + if 'Taxonomy' in kwargs: + kwargs = kwargs['Taxonomy'] super(MISPTaxonomy, self).from_dict(**kwargs) @@ -947,8 +941,8 @@ class MISPGalaxy(AbstractMISP): super(MISPGalaxy, self).__init__() def from_dict(self, **kwargs): - if kwargs.get('Galaxy'): - kwargs = kwargs.get('Galaxy') + if 'Galaxy' in kwargs: + kwargs = kwargs['Galaxy'] super(MISPGalaxy, self).from_dict(**kwargs) @@ -958,8 +952,8 @@ class MISPNoticelist(AbstractMISP): super(MISPNoticelist, self).__init__() def from_dict(self, **kwargs): - if kwargs.get('Noticelist'): - kwargs = kwargs.get('Noticelist') + if 'Noticelist' in kwargs: + kwargs = kwargs['Noticelist'] super(MISPNoticelist, self).from_dict(**kwargs) @@ -969,8 +963,8 @@ class MISPRole(AbstractMISP): super(MISPRole, self).__init__() def from_dict(self, **kwargs): - if kwargs.get('Role'): - kwargs = kwargs.get('Role') + if 'Role' in kwargs: + kwargs = kwargs['Role'] super(MISPRole, self).from_dict(**kwargs) @@ -980,8 +974,8 @@ class MISPServer(AbstractMISP): super(MISPServer, self).__init__() def from_dict(self, **kwargs): - if kwargs.get('Server'): - kwargs = kwargs.get('Server') + if 'Server' in kwargs: + kwargs = kwargs['Server'] super(MISPServer, self).from_dict(**kwargs) @@ -991,8 +985,8 @@ class MISPSharingGroup(AbstractMISP): super(MISPSharingGroup, self).__init__() def from_dict(self, **kwargs): - if kwargs.get('SharingGroup'): - kwargs = kwargs.get('SharingGroup') + if 'SharingGroup' in kwargs: + kwargs = kwargs['SharingGroup'] super(MISPSharingGroup, self).from_dict(**kwargs) @@ -1002,8 +996,8 @@ class MISPLog(AbstractMISP): super(MISPLog, self).__init__() def from_dict(self, **kwargs): - if kwargs.get('Log'): - kwargs = kwargs.get('Log') + if 'Log' in kwargs: + kwargs = kwargs['Log'] super(MISPLog, self).from_dict(**kwargs) def __repr__(self): @@ -1016,8 +1010,8 @@ class MISPEventDelegation(AbstractMISP): super(MISPEventDelegation, self).__init__() def from_dict(self, **kwargs): - if kwargs.get('EventDelegation'): - kwargs = kwargs.get('EventDelegation') + if 'EventDelegation' in kwargs: + kwargs = kwargs['EventDelegation'] super(MISPEventDelegation, self).from_dict(**kwargs) def __repr__(self): @@ -1039,8 +1033,8 @@ class MISPSighting(AbstractMISP): :type: Type of the sighting :timestamp: Timestamp associated to the sighting """ - if kwargs.get('Sighting'): - kwargs = kwargs.get('Sighting') + if 'Sighting' in kwargs: + kwargs = kwargs['Sighting'] super(MISPSighting, self).from_dict(**kwargs) def __repr__(self): @@ -1062,6 +1056,8 @@ class MISPObjectAttribute(MISPAttribute): def from_dict(self, object_relation, value, **kwargs): self.object_relation = object_relation self.value = value + if 'Attribute' in kwargs: + kwargs = kwargs['Attribute'] # Initialize the new MISPAttribute # Get the misp attribute type from the definition self.type = kwargs.pop('type', None) @@ -1078,7 +1074,10 @@ class MISPObjectAttribute(MISPAttribute): self.to_ids = self._definition.get('to_ids') if not self.type: raise NewAttributeError("The type of the attribute is required. Is the object template missing?") - super(MISPObjectAttribute, self).from_dict(**dict(self, **kwargs)) + if sys.version_info < (3, 5): + super(MISPObjectAttribute, self).from_dict(**dict(self, **kwargs)) + else: + super(MISPObjectAttribute, self).from_dict(**{**self, **kwargs}) def __repr__(self): if hasattr(self, 'value'): @@ -1092,8 +1091,8 @@ class MISPShadowAttribute(AbstractMISP): super(MISPShadowAttribute, self).__init__() def from_dict(self, **kwargs): - if kwargs.get('ShadowAttribute'): - kwargs = kwargs.get('ShadowAttribute') + if 'ShadowAttribute' in kwargs: + kwargs = kwargs['ShadowAttribute'] super(MISPShadowAttribute, self).from_dict(**kwargs) def __repr__(self): @@ -1108,8 +1107,8 @@ class MISPCommunity(AbstractMISP): super(MISPCommunity, self).__init__() def from_dict(self, **kwargs): - if kwargs.get('Community'): - kwargs = kwargs.get('Community') + if 'Community' in kwargs: + kwargs = kwargs['Community'] super(MISPCommunity, self).from_dict(**kwargs) def __repr__(self): @@ -1231,8 +1230,8 @@ class MISPObject(AbstractMISP): raise PyMISPError('All the attributes have to be of type MISPObjectReference.') def from_dict(self, **kwargs): - if kwargs.get('Object'): - kwargs = kwargs.get('Object') + if 'Object' in kwargs: + kwargs = kwargs['Object'] if self._known_template: if kwargs.get('template_uuid') and kwargs['template_uuid'] != self.template_uuid: if self._strict: @@ -1260,11 +1259,9 @@ class MISPObject(AbstractMISP): else: self.timestamp = datetime.datetime.fromtimestamp(int(ts), UTC()) if kwargs.get('Attribute'): - for a in kwargs.pop('Attribute'): - self.add_attribute(**a) + [self.add_attribute(**a) for a in kwargs.pop('Attribute')] if kwargs.get('ObjectReference'): - for r in kwargs.pop('ObjectReference'): - self.add_reference(**r) + [self.add_reference(**r) for r in kwargs.pop('ObjectReference')] # Not supported yet - https://github.com/MISP/PyMISP/issues/168 # if kwargs.get('Tag'): @@ -1323,7 +1320,10 @@ class MISPObject(AbstractMISP): else: attribute = MISPObjectAttribute({}) # Overwrite the parameters of self._default_attributes_parameters with the ones of value - attribute.from_dict(object_relation=object_relation, **dict(self._default_attributes_parameters, **value)) + if sys.version_info < (3, 5): + attribute.from_dict(object_relation=object_relation, **dict(self._default_attributes_parameters, **value)) + else: + attribute.from_dict(object_relation=object_relation, **{**self._default_attributes_parameters, **value}) self.__fast_attribute_access[object_relation].append(attribute) self.Attribute.append(attribute) self.edited = True diff --git a/tests/mispevent_testfiles/existing_event.json b/tests/mispevent_testfiles/existing_event.json index 6587dab..40453af 100644 --- a/tests/mispevent_testfiles/existing_event.json +++ b/tests/mispevent_testfiles/existing_event.json @@ -3919,7 +3919,7 @@ "date": "2017-12-14", "distribution": "3", "id": "9616", - "info": "OSINT - Attackers Deploy New ICS Attack Framework “TRITON” and Cause Operational Disruption to Critical Infrastructure", + "info": "OSINT - Attackers Deploy New ICS Attack Framework \"TRITON\" and Cause Operational Disruption to Critical Infrastructure", "org_id": "2", "orgc_id": "2", "published": false, @@ -4019,7 +4019,7 @@ "date": "2017-10-23", "distribution": "3", "id": "9208", - "info": "Talos: “Cyber Conflict” Decoy Document Used In Real Cyber Conflict", + "info": "Talos: \"Cyber Conflict\" Decoy Document Used In Real Cyber Conflict", "org_id": "291", "orgc_id": "291", "published": true, diff --git a/tests/mispevent_testfiles/existing_event_edited.json b/tests/mispevent_testfiles/existing_event_edited.json index 91d0e53..51da02b 100644 --- a/tests/mispevent_testfiles/existing_event_edited.json +++ b/tests/mispevent_testfiles/existing_event_edited.json @@ -3922,7 +3922,7 @@ "date": "2017-12-14", "distribution": "3", "id": "9616", - "info": "OSINT - Attackers Deploy New ICS Attack Framework “TRITON” and Cause Operational Disruption to Critical Infrastructure", + "info": "OSINT - Attackers Deploy New ICS Attack Framework \"TRITON\" and Cause Operational Disruption to Critical Infrastructure", "org_id": "2", "orgc_id": "2", "published": false, @@ -4022,7 +4022,7 @@ "date": "2017-10-23", "distribution": "3", "id": "9208", - "info": "Talos: “Cyber Conflict” Decoy Document Used In Real Cyber Conflict", + "info": "Talos: \"Cyber Conflict\" Decoy Document Used In Real Cyber Conflict", "org_id": "291", "orgc_id": "291", "published": true, diff --git a/tests/test_mispevent.py b/tests/test_mispevent.py index b190541..5f31ca4 100644 --- a/tests/test_mispevent.py +++ b/tests/test_mispevent.py @@ -26,20 +26,20 @@ class TestMISPEvent(unittest.TestCase): def test_simple(self): with open('tests/mispevent_testfiles/simple.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_event(self): self.init_event() self.mispevent.publish() with open('tests/mispevent_testfiles/event.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_loadfile(self): self.mispevent.load_file('tests/mispevent_testfiles/event.json') with open('tests/mispevent_testfiles/event.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_event_tag(self): self.init_event() @@ -50,7 +50,7 @@ class TestMISPEvent(unittest.TestCase): self.mispevent.add_tag(new_tag) with open('tests/mispevent_testfiles/event_tags.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_attribute(self): self.init_event() @@ -62,13 +62,13 @@ class TestMISPEvent(unittest.TestCase): self.assertEqual(attr_tags[0].name, 'osint') with open('tests/mispevent_testfiles/attribute.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) # Fake setting an attribute ID for testing self.mispevent.attributes[0].id = 42 self.mispevent.delete_attribute(42) with open('tests/mispevent_testfiles/attribute_del.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_object_tag(self): self.mispevent.add_object(name='file', strict=True) @@ -90,7 +90,7 @@ class TestMISPEvent(unittest.TestCase): self.assertEqual(self.mispevent.objects[0].references[0].relationship_type, 'baz') with open('tests/mispevent_testfiles/event_obj_attr_tag.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) @unittest.skip("Not supported on MISP: https://github.com/MISP/MISP/issues/2638 - https://github.com/MISP/PyMISP/issues/168") def test_object_level_tag(self): @@ -100,7 +100,7 @@ class TestMISPEvent(unittest.TestCase): self.mispevent.objects[0].uuid = 'a' with open('tests/mispevent_testfiles/event_obj_tag.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_malware(self): with open('tests/mispevent_testfiles/simple.json', 'rb') as f: @@ -112,7 +112,7 @@ class TestMISPEvent(unittest.TestCase): self.assertEqual(attribute.malware_binary, pseudofile) with open('tests/mispevent_testfiles/malware.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_existing_malware(self): self.mispevent.load_file('tests/mispevent_testfiles/malware_exist.json') @@ -127,19 +127,20 @@ class TestMISPEvent(unittest.TestCase): sighting.from_dict(value='1', type='bar', timestamp=11111111) with open('tests/mispevent_testfiles/sighting.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(sighting.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + self.assertEqual(sighting.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_existing_event(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') with open('tests/mispevent_testfiles/existing_event.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + + self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_shadow_attributes_existing(self): self.mispevent.load_file('tests/mispevent_testfiles/shadow.json') with open('tests/mispevent_testfiles/shadow.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) @unittest.skip("Not supported on MISP.") def test_shadow_attributes(self): @@ -152,7 +153,7 @@ class TestMISPEvent(unittest.TestCase): del p.uuid with open('tests/mispevent_testfiles/proposals.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_default_attributes(self): self.mispevent.add_object(name='file', strict=True) @@ -165,7 +166,7 @@ class TestMISPEvent(unittest.TestCase): self.mispevent.objects[1].uuid = 'b' with open('tests/mispevent_testfiles/event_obj_def_param.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_obj_default_values(self): self.init_event() @@ -181,7 +182,7 @@ class TestMISPEvent(unittest.TestCase): self.mispevent.objects[0].uuid = 'a' with open('tests/mispevent_testfiles/def_param.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_event_not_edited(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') @@ -246,7 +247,7 @@ class TestMISPEvent(unittest.TestCase): self.assertTrue(self.mispevent.edited) with open('tests/mispevent_testfiles/existing_event_edited.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_obj_by_id(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') @@ -258,7 +259,7 @@ class TestMISPEvent(unittest.TestCase): self.mispevent.add_object(name='test_object_template', strict=True, misp_objects_path_custom='tests/mispevent_testfiles') with self.assertRaises(InvalidMISPObject) as e: # Fail on required - self.mispevent.to_json() + self.mispevent.to_json(sort_keys=True, indent=2) if sys.version_info >= (3, ): self.assertEqual(e.exception.message, '{\'member3\'} are required.') else: @@ -269,7 +270,7 @@ class TestMISPEvent(unittest.TestCase): del a.uuid with self.assertRaises(InvalidMISPObject) as e: # Fail on requiredOneOf - self.mispevent.to_json() + self.mispevent.to_json(sort_keys=True, indent=2) self.assertEqual(e.exception.message, 'At least one of the following attributes is required: member1, member2') a = self.mispevent.objects[0].add_attribute('member1', value='bar') @@ -278,14 +279,14 @@ class TestMISPEvent(unittest.TestCase): del a.uuid with self.assertRaises(InvalidMISPObject) as e: # member1 is not a multiple - self.mispevent.to_json() + self.mispevent.to_json(sort_keys=True, indent=2) self.assertEqual(e.exception.message, 'Multiple occurrences of member1 is not allowed') self.mispevent.objects[0].attributes = self.mispevent.objects[0].attributes[:2] self.mispevent.objects[0].uuid = 'a' with open('tests/mispevent_testfiles/misp_custom_obj.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) if __name__ == '__main__': From 7c42a5f748c3e6e59a1f16e1759a8baf42b730f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 10 Oct 2019 08:58:06 +0200 Subject: [PATCH 11/12] fix: Python2 SyntaxError... --- pymisp/mispevent.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 574466f..e3a10e8 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -1074,10 +1074,9 @@ class MISPObjectAttribute(MISPAttribute): self.to_ids = self._definition.get('to_ids') if not self.type: raise NewAttributeError("The type of the attribute is required. Is the object template missing?") - if sys.version_info < (3, 5): - super(MISPObjectAttribute, self).from_dict(**dict(self, **kwargs)) - else: - super(MISPObjectAttribute, self).from_dict(**{**self, **kwargs}) + super(MISPObjectAttribute, self).from_dict(**dict(self, **kwargs)) + # FIXME New syntax python3 only, keep for later. + # super(MISPObjectAttribute, self).from_dict(**{**self, **kwargs}) def __repr__(self): if hasattr(self, 'value'): @@ -1320,10 +1319,9 @@ class MISPObject(AbstractMISP): else: attribute = MISPObjectAttribute({}) # Overwrite the parameters of self._default_attributes_parameters with the ones of value - if sys.version_info < (3, 5): - attribute.from_dict(object_relation=object_relation, **dict(self._default_attributes_parameters, **value)) - else: - attribute.from_dict(object_relation=object_relation, **{**self._default_attributes_parameters, **value}) + attribute.from_dict(object_relation=object_relation, **dict(self._default_attributes_parameters, **value)) + # FIXME New syntax python3 only, keep for later. + # attribute.from_dict(object_relation=object_relation, **{**self._default_attributes_parameters, **value}) self.__fast_attribute_access[object_relation].append(attribute) self.Attribute.append(attribute) self.edited = True From 2785d0027d6548be9da7dd9635cd4491dbc6e8ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 10 Oct 2019 10:15:23 +0200 Subject: [PATCH 12/12] fix: [Python2] Use LRU cache decorator, fix call to describe_types in PyMISP --- pymisp/abstract.py | 31 +++++++------------------------ pymisp/api.py | 2 +- tests/test.py | 2 +- 3 files changed, 9 insertions(+), 26 deletions(-) diff --git a/pymisp/abstract.py b/pymisp/abstract.py index 6921d53..0e2d486 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -31,7 +31,7 @@ logger = logging.getLogger('pymisp') if sys.version_info < (3, 0): from collections import MutableMapping import os - import cachetools + from cachetools import cached, LRUCache resources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') misp_objects_path = os.path.join(resources_path, 'misp-objects', 'objects') @@ -55,24 +55,12 @@ if sys.version_info < (3, 0): class MISPFileCache(object): # cache up to 150 JSON structures in class attribute - __file_cache = cachetools.LFUCache(150) - @classmethod - def _load_json(cls, path): - # use root class attribute as global cache - file_cache = cls.__file_cache - # use modified time with path as cache key - mtime = os.path.getmtime(path) - if path in file_cache: - ctime, data = file_cache[path] - if ctime == mtime: - return data - with open(path, 'rb') as f: - if OLD_PY3: - data = loads(f.read().decode()) - else: - data = load(f) - file_cache[path] = (mtime, data) + @staticmethod + @cached(cache=LRUCache(maxsize=150)) + def _load_json(path): + with open(path, 'r') as f: + data = load(f) return data else: @@ -91,15 +79,10 @@ else: @staticmethod @lru_cache(maxsize=150) def _load_json(path): - with path.open('rb') as f: + with path.open('r') as f: data = load(f) return data -if (3, 0) <= sys.version_info < (3, 6): - OLD_PY3 = True -else: - OLD_PY3 = False - class Distribution(Enum): your_organisation_only = 0 diff --git a/pymisp/api.py b/pymisp/api.py index 186cfa3..a5c0670 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -144,7 +144,7 @@ class PyMISP(MISPFileCache): # pragma: no cover if remote_describe_types.get('error'): for e in remote_describe_types.get('error'): raise PyMISPError('Failed: {}'.format(e)) - remote_describe_types = describe_types['result'] + remote_describe_types = describe_types if not remote_describe_types.get('sane_defaults'): raise PyMISPError('The MISP server your are trying to reach is outdated (<2.4.52). Please use PyMISP v2.4.51.1 (pip install -I PyMISP==v2.4.51.1) and/or contact your administrator.') return remote_describe_types diff --git a/tests/test.py b/tests/test.py index 40477ec..45f55e4 100755 --- a/tests/test.py +++ b/tests/test.py @@ -6,7 +6,7 @@ try: except ImportError as e: print(e) url = 'https://localhost:8443' - key = 'K5yV0CcxdnklzDfCKlnPniIxrMX41utQ2dG13zZ3' + key = 'd6OmdDFvU3Seau3UjwvHS1y3tFQbaRNhJhDX0tjh' import time