From f937e844ddb3638a198d4584ace75b7b56984a77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 27 Mar 2018 14:57:07 +0200 Subject: [PATCH] chg: Make object helpers more generic, cleanup. --- examples/add_fail2ban_object.py | 3 +- pymisp/abstract.py | 2 + pymisp/mispevent.py | 62 +++++++++++++++---------------- pymisp/tools/__init__.py | 1 + pymisp/tools/abstractgenerator.py | 28 +++++++++++++- pymisp/tools/fail2banobject.py | 31 +++------------- 6 files changed, 68 insertions(+), 59 deletions(-) diff --git a/examples/add_fail2ban_object.py b/examples/add_fail2ban_object.py index d5931e1..225eed8 100755 --- a/examples/add_fail2ban_object.py +++ b/examples/add_fail2ban_object.py @@ -76,7 +76,8 @@ if __name__ == '__main__': parameters['logline'] = b64decode(args.logline).decode() if args.logfile: with open(args.logfile, 'rb') as f: - parameters['logfile'] = (os.path.basename(args.logfile), BytesIO(f.read())) + parameters['logfile'] = {'value': os.path.basename(args.logfile), + 'data': BytesIO(f.read())} f2b = Fail2BanObject(parameters=parameters, standalone=False) if me: me.add_object(f2b) diff --git a/pymisp/abstract.py b/pymisp/abstract.py index 1036f50..c6411e5 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -39,6 +39,8 @@ class MISPEncode(JSONEncoder): def default(self, obj): if isinstance(obj, AbstractMISP): return obj.jsonable() + elif isinstance(obj, datetime.datetime): + return obj.isoformat() return JSONEncoder.default(self, obj) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 8e718ea..8470fdd 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -792,7 +792,7 @@ class MISPObjectAttribute(MISPAttribute): def __init__(self, definition): super(MISPObjectAttribute, self).__init__() - self.__definition = definition + self._definition = definition def from_dict(self, object_relation, value, **kwargs): self.object_relation = object_relation @@ -801,16 +801,16 @@ class MISPObjectAttribute(MISPAttribute): # Get the misp attribute type from the definition self.type = kwargs.pop('type', None) if self.type is None: - self.type = self.__definition.get('misp-attribute') + self.type = self._definition.get('misp-attribute') self.disable_correlation = kwargs.pop('disable_correlation', None) if self.disable_correlation is None: # The correlation can be disabled by default in the object definition. # Use this value if it isn't overloaded by the object - self.disable_correlation = self.__definition.get('disable_correlation') + self.disable_correlation = self._definition.get('disable_correlation') self.to_ids = kwargs.pop('to_ids', None) if self.to_ids is None: # Same for the to_ids flag - self.to_ids = self.__definition.get('to_ids') + self.to_ids = self._definition.get('to_ids') super(MISPObjectAttribute, self).from_dict(**dict(self, **kwargs)) def __repr__(self): @@ -841,7 +841,7 @@ class MISPObject(AbstractMISP): :misp_objects_path_custom: Path to custom object templates ''' super(MISPObject, self).__init__(**kwargs) - self.__strict = strict + self._strict = strict self.name = name misp_objects_path = os.path.join( os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)), @@ -850,22 +850,22 @@ class MISPObject(AbstractMISP): if misp_objects_path_custom and os.path.exists(os.path.join(misp_objects_path_custom, self.name, 'definition.json')): # Use the local object path by default if provided (allows to overwrite a default template) template_path = os.path.join(misp_objects_path_custom, self.name, 'definition.json') - self.__known_template = True + self._known_template = True elif os.path.exists(os.path.join(misp_objects_path, self.name, 'definition.json')): template_path = os.path.join(misp_objects_path, self.name, 'definition.json') - self.__known_template = True + self._known_template = True else: - if self.__strict: + if self._strict: raise UnknownMISPObjectTemplate('{} is unknown in the MISP object directory.'.format(self.name)) else: - self.__known_template = False - if self.__known_template: + self._known_template = False + if self._known_template: with open(template_path, 'r') as f: - self.__definition = json.load(f) - setattr(self, 'meta-category', self.__definition['meta-category']) - self.template_uuid = self.__definition['uuid'] - self.description = self.__definition['description'] - self.template_version = self.__definition['version'] + self._definition = json.load(f) + setattr(self, 'meta-category', self._definition['meta-category']) + self.template_uuid = self._definition['uuid'] + self.description = self._definition['description'] + self.template_version = self._definition['version'] else: # Then we have no meta-category, template_uuid, description and template_version pass @@ -926,17 +926,17 @@ class MISPObject(AbstractMISP): raise PyMISPError('All the attributes have to be of type MISPObjectReference.') def from_dict(self, **kwargs): - if self.__known_template: + if self._known_template: if kwargs.get('template_uuid') and kwargs['template_uuid'] != self.template_uuid: - if self.__strict: + if self._strict: raise UnknownMISPObjectTemplate('UUID of the object is different from the one of the template.') else: - self.__known_template = False + self._known_template = False if kwargs.get('template_version') and int(kwargs['template_version']) != self.template_version: - if self.__strict: + if self._strict: raise UnknownMISPObjectTemplate('Version of the object ({}) is different from the one of the template ({}).'.format(kwargs['template_version'], self.template_version)) else: - self.__known_template = False + self._known_template = False if kwargs.get('Attribute'): for a in kwargs.pop('Attribute'): @@ -986,9 +986,9 @@ class MISPObject(AbstractMISP): dictionary with all the keys supported by MISPAttribute""" if value.get('value') is None: return None - if self.__known_template: - if self.__definition['attributes'].get(object_relation): - attribute = MISPObjectAttribute(self.__definition['attributes'][object_relation]) + if self._known_template: + if self._definition['attributes'].get(object_relation): + attribute = MISPObjectAttribute(self._definition['attributes'][object_relation]) else: # Woopsie, this object_relation is unknown, no sane defaults for you. logger.warning("The template ({}) doesn't have the object_relation ({}) you're trying to add.".format(self.name, object_relation)) @@ -1003,30 +1003,30 @@ class MISPObject(AbstractMISP): return attribute def to_dict(self, strict=False): - if strict or self.__strict and self.__known_template: + if strict or self._strict and self._known_template: self._validate() return super(MISPObject, self).to_dict() def to_json(self, strict=False): - if strict or self.__strict and self.__known_template: + if strict or self._strict and self._known_template: self._validate() return super(MISPObject, self).to_json() def _validate(self): """Make sure the object we're creating has the required fields""" - if self.__definition.get('required'): - required_missing = set(self.__definition.get('required')) - set(self._fast_attribute_access.keys()) + if self._definition.get('required'): + required_missing = set(self._definition.get('required')) - set(self._fast_attribute_access.keys()) if required_missing: raise InvalidMISPObject('{} are required.'.format(required_missing)) - if self.__definition.get('requiredOneOf'): - if not set(self.__definition['requiredOneOf']) & set(self._fast_attribute_access.keys()): + if self._definition.get('requiredOneOf'): + if not set(self._definition['requiredOneOf']) & set(self._fast_attribute_access.keys()): # We ecpect at least one of the object_relation in requiredOneOf, and it isn't the case - raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self.__definition['requiredOneOf']))) + raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self._definition['requiredOneOf']))) for rel, attrs in self._fast_attribute_access.items(): if len(attrs) == 1: # object_relation's here only once, everything's cool, moving on continue - if not self.__definition['attributes'][rel].get('multiple'): + if not self._definition['attributes'][rel].get('multiple'): # object_relation's here more than once, but it isn't allowed in the template. raise InvalidMISPObject('Multiple occurrences of {} is not allowed'.format(rel)) return True diff --git a/pymisp/tools/__init__.py b/pymisp/tools/__init__.py index 1eaae07..705b2d1 100644 --- a/pymisp/tools/__init__.py +++ b/pymisp/tools/__init__.py @@ -12,6 +12,7 @@ from .genericgenerator import GenericObjectGenerator # noqa from .openioc import load_openioc, load_openioc_file # noqa from .sbsignatureobject import SBSignatureObject # noqa from .fail2banobject import Fail2BanObject # noqa +from .domainipobject import DomainIPObject # noqa if sys.version_info >= (3, 6): from .emailobject import EMailObject # noqa diff --git a/pymisp/tools/abstractgenerator.py b/pymisp/tools/abstractgenerator.py index 2645bb5..b32aa6e 100644 --- a/pymisp/tools/abstractgenerator.py +++ b/pymisp/tools/abstractgenerator.py @@ -4,13 +4,37 @@ import abc import six from .. import MISPObject +from ..exceptions import InvalidMISPObject +from datetime import datetime +from dateutil.parser import parse @six.add_metaclass(abc.ABCMeta) # Remove that line when discarding python2 support. # Python3 way: class MISPObjectGenerator(metaclass=abc.ABCMeta): class AbstractMISPObjectGenerator(MISPObject): - @abc.abstractmethod + def _sanitize_timestamp(self, timestamp): + if not timestamp: + return datetime.now() + elif isinstance(timestamp, dict): + if not isinstance(timestamp['value'], datetime): + timestamp['value'] = parse(timestamp['value']) + return timestamp + elif not isinstance(timestamp, datetime): + return parse(timestamp) + return timestamp + def generate_attributes(self): """Contains the logic where all the values of the object are gathered""" - pass + if hasattr(self, '_parameters'): + for object_relation in self._definition['attributes']: + value = self._parameters.pop(object_relation, None) + if not value: + continue + if isinstance(value, dict): + self.add_attribute(object_relation, **value) + else: + # Assume it is the value only + self.add_attribute(object_relation, value=value) + if self._strict and self._known_template and self._parameters: + raise InvalidMISPObject('Some object relations are unknown in the template and could not be attached: {}'.format(', '.join(self._parameters))) diff --git a/pymisp/tools/fail2banobject.py b/pymisp/tools/fail2banobject.py index c4f542f..e49cd50 100644 --- a/pymisp/tools/fail2banobject.py +++ b/pymisp/tools/fail2banobject.py @@ -1,39 +1,20 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from datetime import datetime from .abstractgenerator import AbstractMISPObjectGenerator import logging -from dateutil.parser import parse logger = logging.getLogger('pymisp') class Fail2BanObject(AbstractMISPObjectGenerator): - def __init__(self, parameters, standalone=True, **kwargs): - super(Fail2BanObject, self).__init__('fail2ban', standalone=standalone, **kwargs) - self.__parameters = parameters + def __init__(self, parameters, strict=True, standalone=True, **kwargs): + super(Fail2BanObject, self).__init__('fail2ban', strict=strict, standalone=standalone, **kwargs) + self._parameters = parameters self.generate_attributes() def generate_attributes(self): - self.add_attribute('banned-ip', value=self.__parameters['banned-ip']) - self.add_attribute('attack-type', value=self.__parameters['attack-type']) - try: - timestamp = parse(self.__parameters['processing-timestamp']) - except Exception: - timestamp = datetime.now() - - self.add_attribute('processing-timestamp', value=timestamp.isoformat()) - - if 'failures' in self.__parameters: - self.add_attribute('failures', value=self.__parameters['failures']) - if 'sensor' in self.__parameters: - self.add_attribute('', value=self.__parameters['sensor']) - if 'victim' in self.__parameters: - self.add_attribute('victim', value=self.__parameters['victim']) - if 'logline' in self.__parameters: - self.add_attribute('logline', value=self.__parameters['logline']) - if 'logfile' in self.__parameters: - self.add_attribute('logfile', value=self.__parameters['logfile'][0], - data=self.__parameters['logfile'][1]) + timestamp = self._sanitize_timestamp(self._parameters.pop('processing-timestamp', None)) + self._parameters['processing-timestamp'] = timestamp + return super(Fail2BanObject, self).generate_attributes()