From 2bc0745fbf9db54adca649479802d0a9a6ec21f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 28 Aug 2017 19:01:53 +0200 Subject: [PATCH] Refactoring in order to load objects --- examples/add_file_object.py | 4 +- examples/generate_file_objects.py | 13 +- pymisp/__init__.py | 14 +-- pymisp/abstract.py | 7 +- pymisp/defaultobjects.py | 195 +++++++++++++++++++++++++++++ pymisp/exceptions.py | 14 +++ pymisp/mispevent.py | 11 +- pymisp/tools/__init__.py | 1 - pymisp/tools/create_misp_object.py | 3 +- pymisp/tools/elfobject.py | 36 +++--- pymisp/tools/fileobject.py | 24 ++-- pymisp/tools/machoobject.py | 30 ++--- pymisp/tools/objectgenerator.py | 142 --------------------- pymisp/tools/peobject.py | 50 ++++---- tests/test_offline.py | 12 +- 15 files changed, 317 insertions(+), 239 deletions(-) create mode 100644 pymisp/defaultobjects.py delete mode 100644 pymisp/tools/objectgenerator.py diff --git a/examples/add_file_object.py b/examples/add_file_object.py index aa8ab8e..71f032b 100755 --- a/examples/add_file_object.py +++ b/examples/add_file_object.py @@ -30,11 +30,11 @@ if __name__ == '__main__': if peo: template_id = pymisp.get_object_template_id(peo.template_uuid) r = pymisp.add_object(args.event, template_id, peo) - for ref in peo.references: + for ref in peo.ObjectReference: r = pymisp.add_object_reference(ref) if fo: template_id = pymisp.get_object_template_id(fo.template_uuid) response = pymisp.add_object(args.event, template_id, fo) - for ref in fo.references: + for ref in fo.ObjectReference: r = pymisp.add_object_reference(ref) diff --git a/examples/generate_file_objects.py b/examples/generate_file_objects.py index 5ff6bb6..3269845 100755 --- a/examples/generate_file_objects.py +++ b/examples/generate_file_objects.py @@ -10,6 +10,7 @@ try: except ImportError: pass + def check(): missing_dependencies = {'pydeep': False, 'lief': False, 'magic': False, 'pymisp': False} try: @@ -38,18 +39,18 @@ def make_objects(path): if seos: for s in seos: to_return['objects'].append(s) - if s.references: - to_return['references'] += s.references + if s.ObjectReference: + to_return['references'] += s.ObjectReference if peo: to_return['objects'].append(peo) - if peo.references: - to_return['references'] += peo.references + if peo.ObjectReference: + to_return['references'] += peo.ObjectReference if fo: to_return['objects'].append(fo) - if fo.references: - to_return['references'] += fo.references + if fo.ObjectReference: + to_return['references'] += fo.ObjectReference return json.dumps(to_return, cls=MISPEncode) if __name__ == '__main__': diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 532dc15..0565fe2 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,12 +1,12 @@ __version__ = '2.4.77' try: - from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey - from .api import PyMISP - from .abstract import AbstractMISP, MISPEncode - from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull - from .tools import Neo4j - from .tools import stix - from .tools import MISPObjectGenerator + from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate # noqa + from .api import PyMISP # noqa + from .abstract import AbstractMISP, MISPEncode # noqa + from .defaultobjects import MISPObject, AbstractMISPObjectGenerator # noqa + from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull # noqa + from .tools import Neo4j # noqa + from .tools import stix # noqa except ImportError: pass diff --git a/pymisp/abstract.py b/pymisp/abstract.py index c1abca6..302a14d 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -5,9 +5,13 @@ import abc import json from json import JSONEncoder import collections - import six # Remove that import when discarding python2 support. +if six.PY2: + import warnings + warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5") + + class MISPEncode(JSONEncoder): def default(self, obj): @@ -17,7 +21,6 @@ class MISPEncode(JSONEncoder): @six.add_metaclass(abc.ABCMeta) # Remove that line when discarding python2 support. -# Python3 way: class MISPObjectGenerator(metaclass=abc.ABCMeta): class AbstractMISP(collections.MutableMapping): attributes = None diff --git a/pymisp/defaultobjects.py b/pymisp/defaultobjects.py new file mode 100644 index 0000000..67ec076 --- /dev/null +++ b/pymisp/defaultobjects.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import abc +import os +import json +import sys +import uuid +from collections import Counter + +from .abstract import AbstractMISP +from .exceptions import UnknownMISPObjectTemplate, InvalidMISPObject +import six # Remove that import when discarding python2 support. + + +if six.PY2: + import warnings + warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5") + + +class MISPObjectReference(AbstractMISP): + + attributes = ['source_uuid', 'destination_uuid', 'relationship_type', 'comment', 'uuid', 'deleted'] + + def __init__(self): + super(MISPObjectReference, self).__init__() + + def from_dict(self, source_uuid, destination_uuid, relationship_type, comment=None, **kwargs): + self.source_uuid = source_uuid + self.destination_uuid = destination_uuid + self.relationship_type = relationship_type + self.comment = comment + for k, v in kwargs: + setattr(self, k, v) + + +class MISPObjectAttribute(AbstractMISP): + + # This list is very limited and hardcoded to fit the current needs (file/pe/pesection creation): MISPAttriute will follow the + # same spec and just add one attribute: object_relation + attributes = ['object_relation', 'value', 'type', 'category', 'disable_correlation', 'to_ids', + 'data', 'encrypt', 'distribution', 'comment', 'uuid', 'event_id'] + + def __init__(self, definition): + super(MISPObjectAttribute, self).__init__() + self.definition = definition + + def from_dict(self, object_relation, value, **kwargs): + from .mispevent import MISPAttribute + self.object_relation = object_relation + self.value = value + # Initialize the new MISPAttribute + # Get the misp attribute type from the definition + self.type = kwargs.pop('type', None) + if self.type is None: + self.type = self.definition.get('misp-attribute') + self.disable_correlation = kwargs.pop('disable_correlation', None) + if self.disable_correlation is None: + # The correlation can be disabled by default in the object definition. + # Use this value if it isn't overloaded by the object + self.disable_correlation = self.definition.get('disable_correlation') + self.to_ids = kwargs.pop('to_ids', None) + if self.to_ids is None: + # Same for the to_ids flag + self.to_ids = self.definition.get('to_ids') + # FIXME: dirty hack until all the classes are ported to the new format but we get the default values + # Initialise rest of the values + for k, v in kwargs.items(): + setattr(self, k, v) + temp_attribute = MISPAttribute() + temp_attribute.set_all_values(**self) + # Update default values + for k, v in temp_attribute.to_dict().items(): + setattr(self, k, v) + + +class MISPObject(AbstractMISP): + + attributes = ['name', 'meta-category', 'uuid', 'description', 'template_version', 'template_uuid', 'Attribute'] + + def __init__(self, name, strict=True): + super(MISPObject, self).__init__() + self.strict = strict + self.name = name + self.misp_objects_path = os.path.join( + os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)), + 'data', 'misp-objects', 'objects') + if os.path.exists(os.path.join(self.misp_objects_path, self.name, 'definition.json')): + self.known_template = True + else: + if self.strict: + raise UnknownMISPObjectTemplate('{} is unknown in the MISP object directory.') + else: + self.known_template = False + if self.known_template: + with open(os.path.join(self.misp_objects_path, self.name, 'definition.json'), 'r') as f: + self.definition = json.load(f) + setattr(self, 'meta-category', self.definition['meta-category']) + self.template_uuid = self.definition['uuid'] + self.description = self.definition['description'] + self.template_version = self.definition['version'] + else: + # FIXME We need to set something for meta-category, template_uuid, description and template_version + pass + self.uuid = str(uuid.uuid4()) + self.Attribute = [] + self.ObjectReference = [] + + def from_dict(self, **kwargs): + if self.known_template: + if kwargs.get('template_uuid') and kwargs['template_uuid'] != self.template_uuid: + if self.strict: + raise UnknownMISPObjectTemplate('UUID of the object is different from the one of the template.') + else: + self.known_template = False + if kwargs.get('template_version') and int(kwargs['template_version']) != self.template_version: + if self.strict: + raise UnknownMISPObjectTemplate('Version of the object ({}) is different from the one of the template ({}).'.format(kwargs['template_version'], self.template_version)) + else: + self.known_template = False + + for key, value in kwargs.items(): + if key == 'Attribute': + for v in value: + self.add_attribute(**v) + elif key == 'ObjectReference': + for v in value: + self.add_reference(**v) + else: + setattr(self, key, value) + + def to_dict(self, strict=True): + if strict or self.strict and self.known_template: + self._validate() + return super(MISPObject, self).to_dict() + + def to_json(self, strict=True): + if strict or self.strict and self.known_template: + self._validate() + return super(MISPObject, self).to_json() + + def _validate(self): + """Make sure the object we're creating has the required fields""" + all_object_relations = [] + for a in self.Attribute: + all_object_relations.append(a.object_relation) + count_relations = dict(Counter(all_object_relations)) + for key, counter in count_relations.items(): + if counter == 1: + continue + if not self.definition['attributes'][key].get('multiple'): + raise InvalidMISPObject('Multiple occurrences of {} is not allowed'.format(key)) + all_attribute_names = set(count_relations.keys()) + if self.definition.get('requiredOneOf'): + if not set(self.definition['requiredOneOf']) & all_attribute_names: + raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self.definition['requiredOneOf']))) + if self.definition.get('required'): + for r in self.definition.get('required'): + if r not in all_attribute_names: + raise InvalidMISPObject('{} is required'.format(r)) + return True + + def add_reference(self, destination_uuid, relationship_type, comment=None, **kwargs): + """Add a link (uuid) to an other object""" + if kwargs.get('source_uuid'): + # Load existing object + source_uuid = kwargs.get('source_uuid') + else: + # New reference + source_uuid = self.uuid + reference = MISPObjectReference() + reference.from_dict(source_uuid=source_uuid, destination_uuid=destination_uuid, + relationship_type=relationship_type, comment=comment, **kwargs) + self.ObjectReference.append(reference) + + def add_attribute(self, object_relation, **value): + if value.get('value') is None: + return None + if self.known_template: + attribute = MISPObjectAttribute(self.definition['attributes'][object_relation]) + else: + attribute = MISPObjectAttribute({}) + attribute.from_dict(object_relation, **value) + self.Attribute.append(attribute) + return attribute + + +@six.add_metaclass(abc.ABCMeta) # Remove that line when discarding python2 support. +# Python3 way: class MISPObjectGenerator(metaclass=abc.ABCMeta): +class AbstractMISPObjectGenerator(MISPObject): + + @abc.abstractmethod + def generate_attributes(self): + """Contains the logic where all the values of the object are gathered""" + pass diff --git a/pymisp/exceptions.py b/pymisp/exceptions.py index f4db340..3ec3d7b 100644 --- a/pymisp/exceptions.py +++ b/pymisp/exceptions.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- + class PyMISPError(Exception): def __init__(self, message): super(PyMISPError, self).__init__(message) @@ -29,3 +30,16 @@ class NoURL(PyMISPError): class NoKey(PyMISPError): pass + + +class MISPObjectException(PyMISPError): + pass + + +class InvalidMISPObject(MISPObjectException): + """Exception raised when an object doesn't respect the contrains in the definition""" + pass + +class UnknownMISPObjectTemplate(MISPObjectException): + """Exception raised when the template is unknown""" + pass diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index d71be70..226b57d 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -11,7 +11,6 @@ import base64 from io import BytesIO from zipfile import ZipFile import hashlib -from .abstract import AbstractMISP try: from dateutil.parser import parse @@ -38,6 +37,7 @@ except ImportError: has_pyme = False from .exceptions import PyMISPError, NewEventError, NewAttributeError +from .defaultobjects import MISPObject # Least dirty way to support python 2 and 3 try: @@ -285,7 +285,7 @@ def _int_to_str(d): return d -class MISPEvent(AbstractMISP): +class MISPEvent(object): def __init__(self, describe_types=None, strict_validation=False): self.ressources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') @@ -338,6 +338,7 @@ class MISPEvent(AbstractMISP): self.RelatedEvent = [] self.Tag = [] self.Galaxy = None + self.Object = None def _serialize(self): return '{date}{threat_level_id}{info}{uuid}{analysis}{timestamp}'.format( @@ -510,6 +511,12 @@ class MISPEvent(AbstractMISP): self.sig = kwargs['sig'] if kwargs.get('global_sig'): self.global_sig = kwargs['global_sig'] + if kwargs.get('Object'): + self.Object = [] + for obj in kwargs['Object']: + tmp_object = MISPObject(obj['name']) + tmp_object.from_dict(**obj) + self.Object.append(tmp_object) def _json(self): # DEPTECATED diff --git a/pymisp/tools/__init__.py b/pymisp/tools/__init__.py index 2a81b2a..3a73be5 100644 --- a/pymisp/tools/__init__.py +++ b/pymisp/tools/__init__.py @@ -1,5 +1,4 @@ from .neo4j import Neo4j # noqa -from .objectgenerator import MISPObjectGenerator, MISPObjectException, InvalidMISPObject # noqa from .fileobject import FileObject # noqa from .peobject import PEObject, PESectionObject # noqa from .elfobject import ELFObject, ELFSectionObject # noqa diff --git a/pymisp/tools/create_misp_object.py b/pymisp/tools/create_misp_object.py index cb18cb0..f6d82c3 100644 --- a/pymisp/tools/create_misp_object.py +++ b/pymisp/tools/create_misp_object.py @@ -1,7 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from pymisp.tools import FileObject, PEObject, ELFObject, MachOObject, MISPObjectException +from pymisp.tools import FileObject, PEObject, ELFObject, MachOObject +from pymisp.exceptions import MISPObjectException import warnings try: diff --git a/pymisp/tools/elfobject.py b/pymisp/tools/elfobject.py index c62a336..872f554 100644 --- a/pymisp/tools/elfobject.py +++ b/pymisp/tools/elfobject.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from pymisp.tools import MISPObjectGenerator +from pymisp.defaultobjects import AbstractMISPObjectGenerator from io import BytesIO from hashlib import md5, sha1, sha256, sha512 import warnings @@ -20,7 +20,7 @@ except ImportError: HAS_PYDEEP = False -class ELFObject(MISPObjectGenerator): +class ELFObject(AbstractMISPObjectGenerator): def __init__(self, parsed=None, filepath=None, pseudofile=None): if not HAS_PYDEEP: @@ -49,10 +49,10 @@ class ELFObject(MISPObjectGenerator): def generate_attributes(self): # General information - self._create_attribute('type', value=str(self.elf.header.file_type).split('.')[1]) - self._create_attribute('entrypoint-address', value=self.elf.entrypoint) - self._create_attribute('arch', value=str(self.elf.header.machine_type).split('.')[1]) - self._create_attribute('os_abi', value=str(self.elf.header.identity_os_abi).split('.')[1]) + self.add_attribute('type', value=str(self.elf.header.file_type).split('.')[1]) + self.add_attribute('entrypoint-address', value=self.elf.entrypoint) + self.add_attribute('arch', value=str(self.elf.header.machine_type).split('.')[1]) + self.add_attribute('os_abi', value=str(self.elf.header.identity_os_abi).split('.')[1]) # Sections self.sections = [] if self.elf.sections: @@ -62,10 +62,10 @@ class ELFObject(MISPObjectGenerator): self.add_reference(s.uuid, 'included-in', 'Section {} of ELF'.format(pos)) pos += 1 self.sections.append(s) - self._create_attribute('number-sections', value=len(self.sections)) + self.add_attribute('number-sections', value=len(self.sections)) -class ELFSectionObject(MISPObjectGenerator): +class ELFSectionObject(AbstractMISPObjectGenerator): def __init__(self, section): # Python3 way @@ -76,16 +76,16 @@ class ELFSectionObject(MISPObjectGenerator): self.generate_attributes() def generate_attributes(self): - self._create_attribute('name', value=self.section.name) - self._create_attribute('type', value=str(self.section.type).split('.')[1]) + self.add_attribute('name', value=self.section.name) + self.add_attribute('type', value=str(self.section.type).split('.')[1]) for flag in self.section.flags_list: - self._create_attribute('flag', value=str(flag).split('.')[1]) - size = self._create_attribute('size-in-bytes', value=self.section.size) + self.add_attribute('flag', value=str(flag).split('.')[1]) + size = self.add_attribute('size-in-bytes', value=self.section.size) if int(size.value) > 0: - self._create_attribute('entropy', value=self.section.entropy) - self._create_attribute('md5', value=md5(self.data).hexdigest()) - self._create_attribute('sha1', value=sha1(self.data).hexdigest()) - self._create_attribute('sha256', value=sha256(self.data).hexdigest()) - self._create_attribute('sha512', value=sha512(self.data).hexdigest()) + self.add_attribute('entropy', value=self.section.entropy) + self.add_attribute('md5', value=md5(self.data).hexdigest()) + self.add_attribute('sha1', value=sha1(self.data).hexdigest()) + self.add_attribute('sha256', value=sha256(self.data).hexdigest()) + self.add_attribute('sha512', value=sha512(self.data).hexdigest()) if HAS_PYDEEP: - self._create_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode()) + self.add_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode()) diff --git a/pymisp/tools/fileobject.py b/pymisp/tools/fileobject.py index ebea362..ed496e5 100644 --- a/pymisp/tools/fileobject.py +++ b/pymisp/tools/fileobject.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from pymisp.tools import MISPObjectGenerator +from pymisp.defaultobjects import AbstractMISPObjectGenerator import os from io import BytesIO from hashlib import md5, sha1, sha256, sha512 @@ -22,7 +22,7 @@ except ImportError: HAS_MAGIC = False -class FileObject(MISPObjectGenerator): +class FileObject(AbstractMISPObjectGenerator): def __init__(self, filepath=None, pseudofile=None, filename=None): if not HAS_PYDEEP: @@ -48,19 +48,19 @@ class FileObject(MISPObjectGenerator): self.generate_attributes() def generate_attributes(self): - self._create_attribute('filename', value=self.filename) - size = self._create_attribute('size-in-bytes', value=len(self.data)) + self.add_attribute('filename', value=self.filename) + size = self.add_attribute('size-in-bytes', value=len(self.data)) if int(size.value) > 0: - self._create_attribute('entropy', value=self.__entropy_H(self.data)) - self._create_attribute('md5', value=md5(self.data).hexdigest()) - self._create_attribute('sha1', value=sha1(self.data).hexdigest()) - self._create_attribute('sha256', value=sha256(self.data).hexdigest()) - self._create_attribute('sha512', value=sha512(self.data).hexdigest()) - self._create_attribute('malware-sample', value=self.filename, data=self.pseudofile) + self.add_attribute('entropy', value=self.__entropy_H(self.data)) + self.add_attribute('md5', value=md5(self.data).hexdigest()) + self.add_attribute('sha1', value=sha1(self.data).hexdigest()) + self.add_attribute('sha256', value=sha256(self.data).hexdigest()) + self.add_attribute('sha512', value=sha512(self.data).hexdigest()) + self.add_attribute('malware-sample', value=self.filename, data=self.pseudofile) if HAS_MAGIC: - self._create_attribute('mimetype', value=magic.from_buffer(self.data)) + self.add_attribute('mimetype', value=magic.from_buffer(self.data)) if HAS_PYDEEP: - self._create_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode()) + self.add_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode()) def __entropy_H(self, data): """Calculate the entropy of a chunk of data.""" diff --git a/pymisp/tools/machoobject.py b/pymisp/tools/machoobject.py index e688fba..53e8e0b 100644 --- a/pymisp/tools/machoobject.py +++ b/pymisp/tools/machoobject.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from pymisp.tools import MISPObjectGenerator +from pymisp.defaultobjects import AbstractMISPObjectGenerator from io import BytesIO from hashlib import md5, sha1, sha256, sha512 import warnings @@ -20,7 +20,7 @@ except ImportError: HAS_PYDEEP = False -class MachOObject(MISPObjectGenerator): +class MachOObject(AbstractMISPObjectGenerator): def __init__(self, parsed=None, filepath=None, pseudofile=None): if not HAS_PYDEEP: @@ -48,11 +48,11 @@ class MachOObject(MISPObjectGenerator): self.generate_attributes() def generate_attributes(self): - self._create_attribute('type', value=str(self.macho.header.file_type).split('.')[1]) - self._create_attribute('name', value=self.macho.name) + self.add_attribute('type', value=str(self.macho.header.file_type).split('.')[1]) + self.add_attribute('name', value=self.macho.name) # General information if self.macho.has_entrypoint: - self._create_attribute('entrypoint-address', value=self.macho.entrypoint) + self.add_attribute('entrypoint-address', value=self.macho.entrypoint) # Sections self.sections = [] if self.macho.sections: @@ -62,10 +62,10 @@ class MachOObject(MISPObjectGenerator): self.add_reference(s.uuid, 'included-in', 'Section {} of MachO'.format(pos)) pos += 1 self.sections.append(s) - self._create_attribute('number-sections', value=len(self.sections)) + self.add_attribute('number-sections', value=len(self.sections)) -class MachOSectionObject(MISPObjectGenerator): +class MachOSectionObject(AbstractMISPObjectGenerator): def __init__(self, section): # Python3 way @@ -76,13 +76,13 @@ class MachOSectionObject(MISPObjectGenerator): self.generate_attributes() def generate_attributes(self): - self._create_attribute('name', value=self.section.name) - size = self._create_attribute('size-in-bytes', value=self.section.size) + self.add_attribute('name', value=self.section.name) + size = self.add_attribute('size-in-bytes', value=self.section.size) if int(size.value) > 0: - self._create_attribute('entropy', value=self.section.entropy) - self._create_attribute('md5', value=md5(self.data).hexdigest()) - self._create_attribute('sha1', value=sha1(self.data).hexdigest()) - self._create_attribute('sha256', value=sha256(self.data).hexdigest()) - self._create_attribute('sha512', value=sha512(self.data).hexdigest()) + self.add_attribute('entropy', value=self.section.entropy) + self.add_attribute('md5', value=md5(self.data).hexdigest()) + self.add_attribute('sha1', value=sha1(self.data).hexdigest()) + self.add_attribute('sha256', value=sha256(self.data).hexdigest()) + self.add_attribute('sha512', value=sha512(self.data).hexdigest()) if HAS_PYDEEP: - self._create_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode()) + self.add_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode()) diff --git a/pymisp/tools/objectgenerator.py b/pymisp/tools/objectgenerator.py deleted file mode 100644 index 46e4143..0000000 --- a/pymisp/tools/objectgenerator.py +++ /dev/null @@ -1,142 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from collections import Counter -from pymisp import MISPEvent, MISPAttribute, AbstractMISP -import os -import json -import uuid -import abc -import sys -import six # Remove that import when discarding python2 support. - - -class MISPObjectException(Exception): - pass - - -class InvalidMISPObject(MISPObjectException): - """Exception raised when an object doesn't contains the required field(s)""" - pass - - -if six.PY2: - import warnings - warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5") - - -class MISPObjectReference(AbstractMISP): - - attributes = ['source_uuid', 'destination_uuid', 'relationship_type', 'comment'] - - def __init__(self, source_uuid, destination_uuid, relationship_type, comment=None): - self.source_uuid = source_uuid - self.destination_uuid = destination_uuid - self.relationship_type = relationship_type - self.comment = comment - - -class MISPObjectAttribute(AbstractMISP): - - # This list is very limited and hardcoded to fit the current needs (file/pe/pesection creation): MISPAttriute will follow the - # same spec and just add one attribute: object_relation - attributes = ['object_relation', 'value', 'type', 'category', 'disable_correlation', 'to_ids', - 'data', 'encrypt', 'distribution', 'comment'] - - def __init__(self, definition, object_relation, value, **kwargs): - self.object_relation = object_relation - self.value = value - # Initialize the new MISPAttribute - # Get the misp attribute type from the definition - self.type = kwargs.pop('type', None) - if self.type is None: - self.type = definition['misp-attribute'] - self.disable_correlation = kwargs.pop('disable_correlation', None) - if self.disable_correlation is None: - # The correlation can be disabled by default in the object definition. - # Use this value if it isn't overloaded by the object - self.disable_correlation = definition.get('disable_correlation') - self.to_ids = kwargs.pop('to_ids', None) - if self.to_ids is None: - # Same for the to_ids flag - self.to_ids = definition.get('to_ids') - # Initialise rest of the values - for k, v in kwargs.items(): - self[k] = v - # FIXME: dirty hack until all the classes are ported to the new format but we get the default values - temp_attribute = MISPAttribute() - temp_attribute.set_all_values(**self) - # Update default values - self.from_dict(**temp_attribute.to_dict()) - - -class MISPObjectGenerator(AbstractMISP): - - attributes = ['name', 'meta-category', 'uuid', 'description', 'version', 'Attribute'] - - def __init__(self, template_dir): - """This class is used to fill a new MISP object with the default values defined in the object template - * template is the path to the template within the misp-object repository - * misp_objects_path is the path to the misp-object repository - """ - self.misp_objects_path = os.path.join( - os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)), - 'data', 'misp-objects', 'objects') - with open(os.path.join(self.misp_objects_path, template_dir, 'definition.json'), 'r') as f: - self.definition = json.load(f) - self.misp_event = MISPEvent() - self.name = self.definition['name'] - setattr(self, 'meta-category', self.definition['meta-category']) - self.template_uuid = self.definition['uuid'] - self.description = self.definition['description'] - self.version = self.definition['version'] - self.uuid = str(uuid.uuid4()) - self.Attribute = [] - self.references = [] - - def _create_attribute(self, object_type, **value): - if value.get('value') is None: - return None - attribute = MISPObjectAttribute(self.definition['attributes'][object_type], object_type, **value) - self.Attribute.append(attribute) - return attribute - - def to_dict(self, strict=True): - if strict: - self._validate() - return super(MISPObjectGenerator, self).to_dict() - - def to_json(self, strict=True): - if strict: - self._validate() - return super(MISPObjectGenerator, self).to_json() - - def _validate(self): - """Make sure the object we're creating has the required fields""" - all_object_relations = [] - for a in self.Attribute: - all_object_relations.append(a.object_relation) - count_relations = dict(Counter(all_object_relations)) - for key, counter in count_relations.items(): - if counter == 1: - continue - if not self.definition['attributes'][key].get('multiple'): - raise InvalidMISPObject('Multiple occurrences of {} is not allowed'.format(key)) - all_attribute_names = set(count_relations.keys()) - if self.definition.get('requiredOneOf'): - if not set(self.definition['requiredOneOf']) & all_attribute_names: - raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self.definition['requiredOneOf']))) - if self.definition.get('required'): - for r in self.definition.get('required'): - if r not in all_attribute_names: - raise InvalidMISPObject('{} is required'.format(r)) - return True - - def add_reference(self, destination_uuid, relationship_type, comment=None): - """Add a link (uuid) to an other object""" - self.references.append(MISPObjectReference(self.uuid, destination_uuid, relationship_type, comment)) - - @abc.abstractmethod - def generate_attributes(self): - """Contains the logic where all the values of the object are gathered""" - pass diff --git a/pymisp/tools/peobject.py b/pymisp/tools/peobject.py index bea117a..71d6795 100644 --- a/pymisp/tools/peobject.py +++ b/pymisp/tools/peobject.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from pymisp.tools import MISPObjectGenerator +from pymisp.defaultobjects import AbstractMISPObjectGenerator from io import BytesIO from hashlib import md5, sha1, sha256, sha512 from datetime import datetime @@ -21,7 +21,7 @@ except ImportError: HAS_PYDEEP = False -class PEObject(MISPObjectGenerator): +class PEObject(AbstractMISPObjectGenerator): def __init__(self, parsed=None, filepath=None, pseudofile=None): if not HAS_PYDEEP: @@ -74,10 +74,10 @@ class PEObject(MISPObjectGenerator): return 'unknown' def generate_attributes(self): - self._create_attribute('type', value=self._get_pe_type()) + self.add_attribute('type', value=self._get_pe_type()) # General information - self._create_attribute('entrypoint-address', value=self.pe.entrypoint) - self._create_attribute('compilation-timestamp', value=datetime.utcfromtimestamp(self.pe.header.time_date_stamps).isoformat()) + self.add_attribute('entrypoint-address', value=self.pe.entrypoint) + self.add_attribute('compilation-timestamp', value=datetime.utcfromtimestamp(self.pe.header.time_date_stamps).isoformat()) # self.imphash = self.pe.get_imphash() try: if (self.pe.has_resources and @@ -85,15 +85,15 @@ class PEObject(MISPObjectGenerator): self.pe.resources_manager.version.has_string_file_info and self.pe.resources_manager.version.string_file_info.langcode_items): fileinfo = dict(self.pe.resources_manager.version.string_file_info.langcode_items[0].items.items()) - self._create_attribute('original-filename', value=fileinfo.get('OriginalFilename')) - self._create_attribute('internal-filename', value=fileinfo.get('InternalName')) - self._create_attribute('file-description', value=fileinfo.get('FileDescription')) - self._create_attribute('file-version', value=fileinfo.get('FileVersion')) - self._create_attribute('lang-id', value=self.pe.resources_manager.version.string_file_info.langcode_items[0].key) - self._create_attribute('product-name', value=fileinfo.get('ProductName')) - self._create_attribute('product-version', value=fileinfo.get('ProductVersion')) - self._create_attribute('company-name', value=fileinfo.get('CompanyName')) - self._create_attribute('legal-copyright', value=fileinfo.get('LegalCopyright')) + self.add_attribute('original-filename', value=fileinfo.get('OriginalFilename')) + self.add_attribute('internal-filename', value=fileinfo.get('InternalName')) + self.add_attribute('file-description', value=fileinfo.get('FileDescription')) + self.add_attribute('file-version', value=fileinfo.get('FileVersion')) + self.add_attribute('lang-id', value=self.pe.resources_manager.version.string_file_info.langcode_items[0].key) + self.add_attribute('product-name', value=fileinfo.get('ProductName')) + self.add_attribute('product-version', value=fileinfo.get('ProductVersion')) + self.add_attribute('company-name', value=fileinfo.get('CompanyName')) + self.add_attribute('legal-copyright', value=fileinfo.get('LegalCopyright')) except lief.read_out_of_bound: # The file is corrupted pass @@ -106,14 +106,14 @@ class PEObject(MISPObjectGenerator): self.add_reference(s.uuid, 'included-in', 'Section {} of PE'.format(pos)) if ((self.pe.entrypoint >= section.virtual_address) and (self.pe.entrypoint < (section.virtual_address + section.virtual_size))): - self._create_attribute('entrypoint-section|position', value='{}|{}'.format(section.name, pos)) + self.add_attribute('entrypoint-section|position', value='{}|{}'.format(section.name, pos)) pos += 1 self.sections.append(s) - self._create_attribute('number-sections', value=len(self.sections)) + self.add_attribute('number-sections', value=len(self.sections)) # TODO: TLSSection / DIRECTORY_ENTRY_TLS -class PESectionObject(MISPObjectGenerator): +class PESectionObject(AbstractMISPObjectGenerator): def __init__(self, section): # Python3 way @@ -124,13 +124,13 @@ class PESectionObject(MISPObjectGenerator): self.generate_attributes() def generate_attributes(self): - self._create_attribute('name', value=self.section.name) - size = self._create_attribute('size-in-bytes', value=self.section.size) + self.add_attribute('name', value=self.section.name) + size = self.add_attribute('size-in-bytes', value=self.section.size) if int(size.value) > 0: - self._create_attribute('entropy', value=self.section.entropy) - self._create_attribute('md5', value=md5(self.data).hexdigest()) - self._create_attribute('sha1', value=sha1(self.data).hexdigest()) - self._create_attribute('sha256', value=sha256(self.data).hexdigest()) - self._create_attribute('sha512', value=sha512(self.data).hexdigest()) + self.add_attribute('entropy', value=self.section.entropy) + self.add_attribute('md5', value=md5(self.data).hexdigest()) + self.add_attribute('sha1', value=sha1(self.data).hexdigest()) + self.add_attribute('sha256', value=sha256(self.data).hexdigest()) + self.add_attribute('sha512', value=sha512(self.data).hexdigest()) if HAS_PYDEEP: - self._create_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode()) + self.add_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode()) diff --git a/tests/test_offline.py b/tests/test_offline.py index 14c5aad..b8eb097 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -230,18 +230,18 @@ class TestOffline(unittest.TestCase): if seos: for s in seos: to_return['objects'].append(s) - if s.references: - to_return['references'] += s.references + if s.ObjectReference: + to_return['references'] += s.ObjectReference if peo: to_return['objects'].append(peo) - if peo.references: - to_return['references'] += peo.references + if peo.ObjectReference: + to_return['references'] += peo.ObjectReference if fo: to_return['objects'].append(fo) - if fo.references: - to_return['references'] += fo.references + if fo.ObjectReference: + to_return['references'] += fo.ObjectReference return json.dumps(to_return, cls=MISPEncode) def test_objects(self, m):