From a497613a85c841b5934174b7f0de57f9893a2dc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 22 Dec 2017 14:49:14 +0100 Subject: [PATCH] chg: Update documentation, cleanup --- pymisp/__init__.py | 21 ++++ pymisp/abstract.py | 22 +++- pymisp/api.py | 29 +---- pymisp/mispevent.py | 259 ++++++++++++++++++++++++-------------------- 4 files changed, 186 insertions(+), 145 deletions(-) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 5de8bee..e9ad297 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,10 +1,31 @@ __version__ = '2.4.84' import sys import logging +import functools +import warnings + logger = logging.getLogger(__name__) FORMAT = "%(levelname)s [%(filename)s:%(lineno)s - %(funcName)s() ] %(message)s" logging.basicConfig(stream=sys.stderr, level=logging.WARNING, format=FORMAT) + +def deprecated(func): + '''This is a decorator which can be used to mark functions + as deprecated. It will result in a warning being emitted + when the function is used.''' + + @functools.wraps(func) + def new_func(*args, **kwargs): + warnings.showwarning( + "Call to deprecated function {}.".format(func.__name__), + category=DeprecationWarning, + filename=func.__code__.co_filename, + lineno=func.__code__.co_firstlineno + 1 + ) + return func(*args, **kwargs) + return new_func + + try: from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate # noqa from .api import PyMISP # noqa diff --git a/pymisp/abstract.py b/pymisp/abstract.py index 6578940..de0c40d 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -30,11 +30,15 @@ class AbstractMISP(collections.MutableMapping): __not_jsonable = [] def __init__(self, **kwargs): + """Abstract class for all the MISP objects""" super(AbstractMISP, self).__init__() - self.__edited = True + self.__edited = True # As we create a new object, we assume it is edited @property def properties(self): + """All the class public properties that will be dumped in the dictionary, and the JSON export. + Note: all the properties starting with a `_` (private), or listed in __not_jsonable will be skipped. + """ to_return = [] for prop, value in vars(self).items(): if prop.startswith('_') or prop in self.__not_jsonable: @@ -43,6 +47,11 @@ class AbstractMISP(collections.MutableMapping): return to_return def from_dict(self, **kwargs): + """Loading all the parameters as class properties, if they aren't `None`. + This method aims to be called when all the properties requiring a special + treatment are processed. + Note: This method is used when you initialize an object with existing data so by default, + the class is flaged as not edited.""" for prop, value in kwargs.items(): if value is None: continue @@ -51,9 +60,11 @@ class AbstractMISP(collections.MutableMapping): self.__edited = False def update_not_jsonable(self, *args): + """Add entries to the __not_jsonable list""" self.__not_jsonable += args def set_not_jsonable(self, *args): + """Set __not_jsonable to a new list""" self.__not_jsonable = args def from_json(self, json_string): @@ -61,6 +72,9 @@ class AbstractMISP(collections.MutableMapping): self.from_dict(json.loads(json_string)) def to_dict(self): + """Dump the lass to a dictionary. + This method automatically removes the timestamp recursively in every object + that has been edited is order to let MISP update the event accordingly.""" to_return = {} for attribute in self.properties: val = getattr(self, attribute, None) @@ -79,9 +93,11 @@ class AbstractMISP(collections.MutableMapping): return to_return def jsonable(self): + """This method is used by the JSON encoder""" return self.to_dict() def to_json(self): + """Dump recursively any class of type MISPAbstract to a json string""" return json.dumps(self, cls=MISPEncode) def __getitem__(self, key): @@ -105,6 +121,8 @@ class AbstractMISP(collections.MutableMapping): @property def edited(self): + """Recursively check if an object has been edited and update the flag accordingly + to the parent objects""" if self.__edited: return self.__edited for p in self.properties: @@ -119,6 +137,7 @@ class AbstractMISP(collections.MutableMapping): @edited.setter def edited(self, val): + """Set the edit flag""" if isinstance(val, bool): self.__edited = val else: @@ -130,6 +149,7 @@ class AbstractMISP(collections.MutableMapping): super(AbstractMISP, self).__setattr__(name, value) def _datetime_to_timestamp(self, d): + """Convert a datetime.datetime object to a timestamp (int)""" if isinstance(d, (int, str)): # Assume we already have a timestamp return d diff --git a/pymisp/api.py b/pymisp/api.py index 9bbb7ae..1d25d38 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -10,13 +10,11 @@ from dateutil.parser import parse import os import base64 import re -import functools import logging -import warnings from io import BytesIO, open import zipfile -from . import __version__ +from . import __version__, deprecated from .exceptions import PyMISPError, SearchError, NoURL, NoKey from .mispevent import MISPEvent, MISPAttribute, MISPUser, MISPOrganisation from .abstract import MISPEncode @@ -46,23 +44,6 @@ except ImportError: ASYNC_OK = False -def deprecated(func): - '''This is a decorator which can be used to mark functions - as deprecated. It will result in a warning being emitted - when the function is used.''' - - @functools.wraps(func) - def new_func(*args, **kwargs): - warnings.showwarning( - "Call to deprecated function {}.".format(func.__name__), - category=DeprecationWarning, - filename=func.__code__.co_filename, - lineno=func.__code__.co_firstlineno + 1 - ) - return func(*args, **kwargs) - return new_func - - class PyMISP(object): """Python API for MISP @@ -282,8 +263,8 @@ class PyMISP(object): def _prepare_full_event(self, distribution, threat_level_id, analysis, info, date=None, published=False, orgc_id=None, org_id=None, sharing_group_id=None): """Initialize a new MISPEvent from scratch""" misp_event = MISPEvent(self.describe_types) - misp_event.set_all_values(info=info, distribution=distribution, threat_level_id=threat_level_id, - analysis=analysis, date=date, orgc_id=orgc_id, org_id=org_id, sharing_group_id=sharing_group_id) + misp_event.from_dict(info=info, distribution=distribution, threat_level_id=threat_level_id, + analysis=analysis, date=date, orgc_id=orgc_id, org_id=org_id, sharing_group_id=sharing_group_id) if published: misp_event.publish() return misp_event @@ -291,8 +272,8 @@ class PyMISP(object): def _prepare_full_attribute(self, category, type_value, value, to_ids, comment=None, distribution=None, **kwargs): """Initialize a new MISPAttribute from scratch""" misp_attribute = MISPAttribute(self.describe_types) - misp_attribute.set_all_values(type=type_value, value=value, category=category, - to_ids=to_ids, comment=comment, distribution=distribution, **kwargs) + misp_attribute.from_dict(type=type_value, value=value, category=category, + to_ids=to_ids, comment=comment, distribution=distribution, **kwargs) return misp_attribute def _valid_uuid(self, uuid): diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 9b68771..55fc57c 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -12,6 +12,7 @@ import sys import uuid from collections import Counter +from . import deprecated from .abstract import AbstractMISP from .exceptions import UnknownMISPObjectTemplate, InvalidMISPObject, PyMISPError, NewEventError, NewAttributeError @@ -102,9 +103,11 @@ class MISPAttribute(AbstractMISP): raise PyMISPError('All the attributes have to be of type MISPAttribute.') def delete(self): + """Mark the attribute as deleted (soft delete)""" self.deleted = True def add_tag(self, tag): + """Add a tag to the attribute (by name or a MISPTag object)""" misp_tag = MISPTag() if isinstance(tag, str): misp_tag.from_dict(name=tag) @@ -115,11 +118,6 @@ class MISPAttribute(AbstractMISP): self.tags.append(misp_tag) self.edited = True - def __repr__(self): - if hasattr(self, 'value'): - return '<{self.__class__.__name__}(type={self.type}, value={self.value})'.format(self=self) - return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) - def from_dict(self, **kwargs): if kwargs.get('type') and kwargs.get('category'): if kwargs['type'] not in self.__category_type_mapping[kwargs['category']]: @@ -129,7 +127,7 @@ class MISPAttribute(AbstractMISP): self.type = kwargs.pop('type', None) # Required if self.type is None: raise NewAttributeError('The type of the attribute is required.') - if self.type not in self.get_known_types(): + if self.type not in self.known_types: raise NewAttributeError('{} is invalid, type has to be in {}'.format(self.type, (', '.join(self._types)))) type_defaults = self.__sane_default[self.type] @@ -184,6 +182,13 @@ class MISPAttribute(AbstractMISP): super(MISPAttribute, self).from_dict(**kwargs) + def to_dict(self): + to_return = super(MISPAttribute, self).to_dict() + if to_return.get('data'): + to_return['data'] = base64.b64encode(self.data.getvalue()).decode() + to_return = _int_to_str(to_return) + return to_return + def _prepare_new_malware_sample(self): if '|' in self.value: # Get the filename, ignore the md5, because humans. @@ -233,12 +238,10 @@ class MISPAttribute(AbstractMISP): # not a encrypted zip file, assuming it is a new malware sample self._prepare_new_malware_sample() - def to_dict(self): - to_return = super(MISPAttribute, self).to_dict() - if to_return.get('data'): - to_return['data'] = base64.b64encode(self.data.getvalue()).decode() - to_return = _int_to_str(to_return) - return to_return + def __repr__(self): + if hasattr(self, 'value'): + return '<{self.__class__.__name__}(type={self.type}, value={self.value})'.format(self=self) + return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) def verify(self, gpg_uid): # Not used @@ -272,24 +275,24 @@ class MISPAttribute(AbstractMISP): signed, _ = c.sign(to_sign, mode=mode.DETACH) self.sig = base64.b64encode(signed).decode() + @deprecated def get_known_types(self): - # Deprecated return self.known_types + @deprecated def get_malware_binary(self): - # Deprecated return self.malware_binary + @deprecated def _json(self): - # DEPRECATED return self.to_dict() + @deprecated def _json_full(self): - # Deprecated return self.to_dict() + @deprecated def set_all_values(self, **kwargs): - # Deprecated self.from_dict(**kwargs) @@ -319,13 +322,38 @@ class MISPEvent(AbstractMISP): def known_types(self): return self._types + @property + def attributes(self): + return self.Attribute + + @attributes.setter + def attributes(self, attributes): + if all(isinstance(x, MISPAttribute) for x in attributes): + self.Attribute = attributes + else: + raise PyMISPError('All the attributes have to be of type MISPAttribute.') + + @property + def related_events(self): + return self.RelatedEvent + + @property + def objects(self): + return self.Object + + @property + def tags(self): + return self.Tag + def load_file(self, event_path): + """Load a JSON dump from a file on the disk""" if not os.path.exists(event_path): raise PyMISPError('Invalid path, unable to load the event.') with open(event_path, 'r') as f: self.load(f) def load(self, json_event): + """Load a JSON dump from a pseudo file or a JSON string""" if hasattr(json_event, 'read'): # python2 and python3 compatible to find if we have a file json_event = json_event.read() @@ -342,9 +370,10 @@ class MISPEvent(AbstractMISP): event['Event']['attribute_count'] = '0' jsonschema.validate(event, self.__json_schema) e = event.get('Event') - self.set_all_values(**e) + self.from_dict(**e) def set_date(self, date, ignore_invalid=False): + """Set a date for the event (string, datetime, or date object)""" if isinstance(date, basestring) or isinstance(date, unicode): self.date = parse(date).date() elif isinstance(date, datetime.datetime): @@ -357,11 +386,6 @@ class MISPEvent(AbstractMISP): else: raise NewEventError('Invalid format for the date: {} - {}'.format(date, type(date))) - def __repr__(self): - if hasattr(self, 'info'): - return '<{self.__class__.__name__}(info={self.info})'.format(self=self) - return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) - def from_dict(self, **kwargs): # Required value self.info = kwargs.pop('info', None) @@ -396,7 +420,7 @@ class MISPEvent(AbstractMISP): if kwargs.get('Attribute'): for a in kwargs.pop('Attribute'): attribute = MISPAttribute() - attribute.set_all_values(**a) + attribute.from_dict(**a) if not hasattr(self, 'Attribute'): self.Attribute = [] self.Attribute.append(attribute) @@ -448,6 +472,7 @@ class MISPEvent(AbstractMISP): return to_return def add_tag(self, tag): + """Add a tag to the attribute (by name or a MISPTag object)""" misp_tag = MISPTag() if isinstance(tag, str): misp_tag.from_dict(name=tag) @@ -490,12 +515,15 @@ class MISPEvent(AbstractMISP): return attributes def publish(self): + """Mark the attribute as published""" self.published = True def unpublish(self): + """Mark the attribute as un-published (set publish flag to false)""" self.published = False def delete_attribute(self, attribute_id): + """Delete an attribute, you can search by ID or UUID""" found = False for a in self.attributes: if ((hasattr(a, 'id') and a.id == attribute_id) or @@ -507,47 +535,28 @@ class MISPEvent(AbstractMISP): raise Exception('No attribute with UUID/ID {} found.'.format(attribute_id)) def add_attribute(self, type, value, **kwargs): + """Add an attribute. type and value are required but you can pass all + other parameters supported by MISPAttribute""" if isinstance(value, list): for a in value: self.add_attribute(type=type, value=a, **kwargs) else: attribute = MISPAttribute() - attribute.set_all_values(type=type, value=value, **kwargs) + attribute.from_dict(type=type, value=value, **kwargs) if not hasattr(self, 'attributes'): self.attributes = [] self.attributes.append(attribute) self.edited = True - @property - def attributes(self): - return self.Attribute - - @attributes.setter - def attributes(self, attributes): - if all(isinstance(x, MISPAttribute) for x in attributes): - self.Attribute = attributes - else: - raise PyMISPError('All the attributes have to be of type MISPAttribute.') - - @property - def related_events(self): - return self.RelatedEvent - - @property - def objects(self): - return self.Object - - @property - def tags(self): - return self.Tag - def get_object_by_id(self, object_id): + """Get an object by ID (the ID is the one set by the server when creating the new object)""" for obj in self.objects: if hasattr(obj, 'id') and obj.id == object_id: return obj raise InvalidMISPObject('Object with {} does not exists in ths event'.format(object_id)) def add_object(self, obj): + """Add an object to the Event, either by passing a MISPObject, or a dictionary""" if isinstance(obj, MISPObject): self.Object.append(obj) elif isinstance(obj, dict): @@ -558,18 +567,25 @@ class MISPEvent(AbstractMISP): raise InvalidMISPObject("An object to add to an existing Event needs to be either a MISPObject, or a plain python dictionary") self.edited = True + def __repr__(self): + if hasattr(self, 'info'): + return '<{self.__class__.__name__}(info={self.info})'.format(self=self) + return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) + def _serialize(self): return '{date}{threat_level_id}{info}{uuid}{analysis}{timestamp}'.format( date=self.date, threat_level_id=self.threat_level_id, info=self.info, uuid=self.uuid, analysis=self.analysis, timestamp=self.timestamp).encode() def _serialize_sigs(self): + # Not used all_sigs = self.sig for a in self.attributes: all_sigs += a.sig return all_sigs.encode() def sign(self, gpg_uid, passphrase=None): + # Not used if not has_pyme: raise PyMISPError('pyme is required, please install: pip install --pre pyme3. You will also need libgpg-error-dev and libgpgme11-dev.') to_sign = self._serialize() @@ -592,6 +608,7 @@ class MISPEvent(AbstractMISP): self.global_sig = base64.b64encode(signed).decode() def verify(self, gpg_uid): + # Not used if not has_pyme: raise PyMISPError('pyme is required, please install: pip install --pre pyme3. You will also need libgpg-error-dev and libgpgme11-dev.') to_return = {} @@ -615,16 +632,16 @@ class MISPEvent(AbstractMISP): to_return['global'] = False return to_return + @deprecated def get_known_types(self): - # Deprecated return self.known_types + @deprecated def set_all_values(self, **kwargs): - # Deprecated self.from_dict(**kwargs) + @deprecated def _json(self): - # DEPTECATED return self.to_dict() @@ -632,26 +649,21 @@ class MISPTag(AbstractMISP): def __init__(self): super(MISPTag, self).__init__() + def from_dict(self, name, **kwargs): + self.name = name + super(MISPTag, self).from_dict(**kwargs) + def __repr__(self): if hasattr(self, 'name'): return '<{self.__class__.__name__}(name={self.name})'.format(self=self) return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) - def from_dict(self, name, **kwargs): - self.name = name - super(MISPTag, self).from_dict(**kwargs) - class MISPObjectReference(AbstractMISP): def __init__(self): super(MISPObjectReference, self).__init__() - def __repr__(self): - if hasattr(self, 'referenced_uuid'): - return '<{self.__class__.__name__}(object_uuid={self.object_uuid}, referenced_uuid={self.referenced_uuid}, relationship_type={self.relationship_type})'.format(self=self) - return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) - def from_dict(self, object_uuid, referenced_uuid, relationship_type, comment=None, **kwargs): self.object_uuid = object_uuid self.referenced_uuid = referenced_uuid @@ -659,6 +671,11 @@ class MISPObjectReference(AbstractMISP): self.comment = comment super(MISPObjectReference, self).from_dict(**kwargs) + def __repr__(self): + if hasattr(self, 'referenced_uuid'): + return '<{self.__class__.__name__}(object_uuid={self.object_uuid}, referenced_uuid={self.referenced_uuid}, relationship_type={self.relationship_type})'.format(self=self) + return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) + class MISPUser(AbstractMISP): @@ -678,11 +695,6 @@ class MISPObjectAttribute(MISPAttribute): super(MISPObjectAttribute, self).__init__() self.__definition = definition - def __repr__(self): - if hasattr(self, 'value'): - return '<{self.__class__.__name__}(object_relation={self.object_relation}, value={self.value})'.format(self=self) - return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) - def from_dict(self, object_relation, value, **kwargs): self.object_relation = object_relation self.value = value @@ -703,6 +715,11 @@ class MISPObjectAttribute(MISPAttribute): kwargs.update(**self) super(MISPObjectAttribute, self).from_dict(**kwargs) + def __repr__(self): + if hasattr(self, 'value'): + return '<{self.__class__.__name__}(object_relation={self.object_relation}, value={self.value})'.format(self=self) + return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) + class MISPObject(AbstractMISP): @@ -760,7 +777,7 @@ class MISPObject(AbstractMISP): self.distribution = self._default_attributes_parameters.distribution self.sharing_group_id = self._default_attributes_parameters.sharing_group_id else: - self.distribution = 3 + self.distribution = 5 # Default to inherit self.sharing_group_id = None self.ObjectReference = [] self._standalone = standalone @@ -768,10 +785,13 @@ class MISPObject(AbstractMISP): # Mark as non_jsonable because we need to add the references manually after the object(s) have been created self.update_not_jsonable('ObjectReference') - def __repr__(self): - if hasattr(self, 'name'): - return '<{self.__class__.__name__}(name={self.name})'.format(self=self) - return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) + @property + def attributes(self): + return self.Attribute + + @property + def references(self): + return self.ObjectReference def from_dict(self, **kwargs): if self.__known_template: @@ -795,6 +815,51 @@ class MISPObject(AbstractMISP): super(MISPObject, self).from_dict(**kwargs) + def add_reference(self, referenced_uuid, relationship_type, comment=None, **kwargs): + """Add a link (uuid) to an other object""" + if kwargs.get('object_uuid'): + # Load existing object + object_uuid = kwargs.pop('object_uuid') + else: + # New reference + object_uuid = self.uuid + reference = MISPObjectReference() + reference.from_dict(object_uuid=object_uuid, referenced_uuid=referenced_uuid, + relationship_type=relationship_type, comment=comment, **kwargs) + self.ObjectReference.append(reference) + self.edited = True + + def get_attributes_by_relation(self, object_relation): + '''Returns the list of attributes with the given object relation in the object''' + return self.__fast_attribute_access.get(object_relation, []) + + def has_attributes_by_relation(self, list_of_relations): + '''True if all the relations in the list are defined in the object''' + return all(relation in self.__fast_attribute_access for relation in list_of_relations) + + def add_attribute(self, object_relation, **value): + """Add an attribute. object_relation is required and the value key is a + dictionary with all the keys supported by MISPAttribute""" + if value.get('value') is None: + return None + if self.__known_template: + if self.__definition['attributes'].get(object_relation): + attribute = MISPObjectAttribute(self.__definition['attributes'][object_relation]) + else: + # Woopsie, this object_relation is unknown, no sane defaults for you. + logger.warning("The template ({}) doesn't have the object_relation ({}) you're trying to add.".format(self.name, object_relation)) + attribute = MISPObjectAttribute({}) + else: + attribute = MISPObjectAttribute({}) + # Overwrite the parameters of self._default_attributes_parameters with the ones of value + attribute.from_dict(object_relation=object_relation, **dict(self._default_attributes_parameters, **value)) + if not self.__fast_attribute_access.get(object_relation): + self.__fast_attribute_access[object_relation] = [] + self.Attribute.append(attribute) + self.__fast_attribute_access[object_relation].append(attribute) + self.edited = True + return attribute + def to_dict(self, strict=False): if strict or self.__strict and self.__known_template: self._validate() @@ -826,53 +891,7 @@ class MISPObject(AbstractMISP): raise InvalidMISPObject('{} is required'.format(r)) return True - def add_reference(self, referenced_uuid, relationship_type, comment=None, **kwargs): - """Add a link (uuid) to an other object""" - if kwargs.get('object_uuid'): - # Load existing object - object_uuid = kwargs.pop('object_uuid') - else: - # New reference - object_uuid = self.uuid - reference = MISPObjectReference() - reference.from_dict(object_uuid=object_uuid, referenced_uuid=referenced_uuid, - relationship_type=relationship_type, comment=comment, **kwargs) - self.ObjectReference.append(reference) - self.edited = True - - def get_attributes_by_relation(self, object_relation): - '''Returns the list of attributes with the given object relation in the object''' - return self.__fast_attribute_access.get(object_relation, []) - - def has_attributes_by_relation(self, list_of_relations): - '''True if all the relations in the list are defined in the object''' - return all(relation in self.__fast_attribute_access for relation in list_of_relations) - - @property - def attributes(self): - return self.Attribute - - @property - def references(self): - return self.ObjectReference - - def add_attribute(self, object_relation, **value): - if value.get('value') is None: - return None - if self.__known_template: - if self.__definition['attributes'].get(object_relation): - attribute = MISPObjectAttribute(self.__definition['attributes'][object_relation]) - else: - # Woopsie, this object_relation is unknown, no sane defaults for you. - logger.warning("The template ({}) doesn't have the object_relation ({}) you're trying to add.".format(self.name, object_relation)) - attribute = MISPObjectAttribute({}) - else: - attribute = MISPObjectAttribute({}) - # Overwrite the parameters of self._default_attributes_parameters with the ones of value - attribute.from_dict(object_relation=object_relation, **dict(self._default_attributes_parameters, **value)) - if not self.__fast_attribute_access.get(object_relation): - self.__fast_attribute_access[object_relation] = [] - self.Attribute.append(attribute) - self.__fast_attribute_access[object_relation].append(attribute) - self.edited = True - return attribute + def __repr__(self): + if hasattr(self, 'name'): + return '<{self.__class__.__name__}(name={self.name})'.format(self=self) + return '<{self.__class__.__name__}(NotInitialized)'.format(self=self)