Refactoring in order to load objects

examples/add_file_object.py

@@ -30,11 +30,11 @@ if __name__ == '__main__':
         if peo:
             template_id = pymisp.get_object_template_id(peo.template_uuid)
             r = pymisp.add_object(args.event, template_id, peo)
-            for ref in peo.references:
+            for ref in peo.ObjectReference:
                 r = pymisp.add_object_reference(ref)
 
         if fo:
             template_id = pymisp.get_object_template_id(fo.template_uuid)
             response = pymisp.add_object(args.event, template_id, fo)
-            for ref in fo.references:
+            for ref in fo.ObjectReference:
                 r = pymisp.add_object_reference(ref)

examples/generate_file_objects.py

@@ -10,6 +10,7 @@ try:
 except ImportError:
     pass
 
+
 def check():
     missing_dependencies = {'pydeep': False, 'lief': False, 'magic': False, 'pymisp': False}
     try:
@@ -38,18 +39,18 @@ def make_objects(path):
     if seos:
         for s in seos:
             to_return['objects'].append(s)
-            if s.references:
+            if s.ObjectReference:
-                to_return['references'] += s.references
+                to_return['references'] += s.ObjectReference
 
     if peo:
         to_return['objects'].append(peo)
-        if peo.references:
+        if peo.ObjectReference:
-            to_return['references'] += peo.references
+            to_return['references'] += peo.ObjectReference
 
     if fo:
         to_return['objects'].append(fo)
-        if fo.references:
+        if fo.ObjectReference:
-            to_return['references'] += fo.references
+            to_return['references'] += fo.ObjectReference
     return json.dumps(to_return, cls=MISPEncode)
 
 if __name__ == '__main__':

pymisp/__init__.py

@@ -1,12 +1,12 @@
 __version__ = '2.4.77'
 
 try:
-    from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey
+    from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate  # noqa
-    from .api import PyMISP
+    from .api import PyMISP  # noqa
-    from .abstract import AbstractMISP, MISPEncode
+    from .abstract import AbstractMISP, MISPEncode  # noqa
-    from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull
+    from .defaultobjects import MISPObject, AbstractMISPObjectGenerator  # noqa
-    from .tools import Neo4j
+    from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull  # noqa
-    from .tools import stix
+    from .tools import Neo4j  # noqa
-    from .tools import MISPObjectGenerator
+    from .tools import stix  # noqa
 except ImportError:
     pass

pymisp/abstract.py

@@ -5,9 +5,13 @@ import abc
 import json
 from json import JSONEncoder
 import collections
-
 import six  # Remove that import when discarding python2 support.
 
+if six.PY2:
+    import warnings
+    warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5")
+
+
 class MISPEncode(JSONEncoder):
 
     def default(self, obj):
@@ -17,7 +21,6 @@ class MISPEncode(JSONEncoder):
 
 
 @six.add_metaclass(abc.ABCMeta)   # Remove that line when discarding python2 support.
-# Python3 way: class MISPObjectGenerator(metaclass=abc.ABCMeta):
 class AbstractMISP(collections.MutableMapping):
 
     attributes = None

pymisp/defaultobjects.py

@@ -0,0 +1,195 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import abc
+import os
+import json
+import sys
+import uuid
+from collections import Counter
+
+from .abstract import AbstractMISP
+from .exceptions import UnknownMISPObjectTemplate, InvalidMISPObject
+import six  # Remove that import when discarding python2 support.
+
+
+if six.PY2:
+    import warnings
+    warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5")
+
+
+class MISPObjectReference(AbstractMISP):
+
+    attributes = ['source_uuid', 'destination_uuid', 'relationship_type', 'comment', 'uuid', 'deleted']
+
+    def __init__(self):
+        super(MISPObjectReference, self).__init__()
+
+    def from_dict(self, source_uuid, destination_uuid, relationship_type, comment=None, **kwargs):
+        self.source_uuid = source_uuid
+        self.destination_uuid = destination_uuid
+        self.relationship_type = relationship_type
+        self.comment = comment
+        for k, v in kwargs:
+            setattr(self, k, v)
+
+
+class MISPObjectAttribute(AbstractMISP):
+
+    # This list is very limited and hardcoded to fit the current needs (file/pe/pesection creation): MISPAttriute will follow the
+    # same spec and just add one attribute: object_relation
+    attributes = ['object_relation', 'value', 'type', 'category', 'disable_correlation', 'to_ids',
+                  'data', 'encrypt', 'distribution', 'comment', 'uuid', 'event_id']
+
+    def __init__(self, definition):
+        super(MISPObjectAttribute, self).__init__()
+        self.definition = definition
+
+    def from_dict(self, object_relation, value, **kwargs):
+        from .mispevent import MISPAttribute
+        self.object_relation = object_relation
+        self.value = value
+        # Initialize the new MISPAttribute
+        # Get the misp attribute type from the definition
+        self.type = kwargs.pop('type', None)
+        if self.type is None:
+            self.type = self.definition.get('misp-attribute')
+        self.disable_correlation = kwargs.pop('disable_correlation', None)
+        if self.disable_correlation is None:
+            # The correlation can be disabled by default in the object definition.
+            # Use this value if it isn't overloaded by the object
+            self.disable_correlation = self.definition.get('disable_correlation')
+        self.to_ids = kwargs.pop('to_ids', None)
+        if self.to_ids is None:
+            # Same for the to_ids flag
+            self.to_ids = self.definition.get('to_ids')
+        # FIXME: dirty hack until all the classes are ported to the new format but we get the default values
+        # Initialise rest of the values
+        for k, v in kwargs.items():
+            setattr(self, k, v)
+        temp_attribute = MISPAttribute()
+        temp_attribute.set_all_values(**self)
+        # Update default values
+        for k, v in temp_attribute.to_dict().items():
+            setattr(self, k, v)
+
+
+class MISPObject(AbstractMISP):
+
+    attributes = ['name', 'meta-category', 'uuid', 'description', 'template_version', 'template_uuid', 'Attribute']
+
+    def __init__(self, name, strict=True):
+        super(MISPObject, self).__init__()
+        self.strict = strict
+        self.name = name
+        self.misp_objects_path = os.path.join(
+            os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)),
+            'data', 'misp-objects', 'objects')
+        if os.path.exists(os.path.join(self.misp_objects_path, self.name, 'definition.json')):
+            self.known_template = True
+        else:
+            if self.strict:
+                raise UnknownMISPObjectTemplate('{} is unknown in the MISP object directory.')
+            else:
+                self.known_template = False
+        if self.known_template:
+            with open(os.path.join(self.misp_objects_path, self.name, 'definition.json'), 'r') as f:
+                self.definition = json.load(f)
+            setattr(self, 'meta-category', self.definition['meta-category'])
+            self.template_uuid = self.definition['uuid']
+            self.description = self.definition['description']
+            self.template_version = self.definition['version']
+        else:
+            # FIXME We need to set something for meta-category, template_uuid, description and template_version
+            pass
+        self.uuid = str(uuid.uuid4())
+        self.Attribute = []
+        self.ObjectReference = []
+
+    def from_dict(self, **kwargs):
+        if self.known_template:
+            if kwargs.get('template_uuid') and kwargs['template_uuid'] != self.template_uuid:
+                if self.strict:
+                    raise UnknownMISPObjectTemplate('UUID of the object is different from the one of the template.')
+                else:
+                    self.known_template = False
+            if kwargs.get('template_version') and int(kwargs['template_version']) != self.template_version:
+                if self.strict:
+                    raise UnknownMISPObjectTemplate('Version of the object ({}) is different from the one of the template ({}).'.format(kwargs['template_version'], self.template_version))
+                else:
+                    self.known_template = False
+
+        for key, value in kwargs.items():
+            if key == 'Attribute':
+                for v in value:
+                    self.add_attribute(**v)
+            elif key == 'ObjectReference':
+                for v in value:
+                    self.add_reference(**v)
+            else:
+                setattr(self, key, value)
+
+    def to_dict(self, strict=True):
+        if strict or self.strict and self.known_template:
+            self._validate()
+        return super(MISPObject, self).to_dict()
+
+    def to_json(self, strict=True):
+        if strict or self.strict and self.known_template:
+            self._validate()
+        return super(MISPObject, self).to_json()
+
+    def _validate(self):
+        """Make sure the object we're creating has the required fields"""
+        all_object_relations = []
+        for a in self.Attribute:
+            all_object_relations.append(a.object_relation)
+        count_relations = dict(Counter(all_object_relations))
+        for key, counter in count_relations.items():
+            if counter == 1:
+                continue
+            if not self.definition['attributes'][key].get('multiple'):
+                raise InvalidMISPObject('Multiple occurrences of {} is not allowed'.format(key))
+        all_attribute_names = set(count_relations.keys())
+        if self.definition.get('requiredOneOf'):
+            if not set(self.definition['requiredOneOf']) & all_attribute_names:
+                raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self.definition['requiredOneOf'])))
+        if self.definition.get('required'):
+            for r in self.definition.get('required'):
+                if r not in all_attribute_names:
+                    raise InvalidMISPObject('{} is required'.format(r))
+        return True
+
+    def add_reference(self, destination_uuid, relationship_type, comment=None, **kwargs):
+        """Add a link (uuid) to an other object"""
+        if kwargs.get('source_uuid'):
+            # Load existing object
+            source_uuid = kwargs.get('source_uuid')
+        else:
+            # New reference
+            source_uuid = self.uuid
+        reference = MISPObjectReference()
+        reference.from_dict(source_uuid=source_uuid, destination_uuid=destination_uuid,
+                            relationship_type=relationship_type, comment=comment, **kwargs)
+        self.ObjectReference.append(reference)
+
+    def add_attribute(self, object_relation, **value):
+        if value.get('value') is None:
+            return None
+        if self.known_template:
+            attribute = MISPObjectAttribute(self.definition['attributes'][object_relation])
+        else:
+            attribute = MISPObjectAttribute({})
+        attribute.from_dict(object_relation, **value)
+        self.Attribute.append(attribute)
+        return attribute
+
+
+@six.add_metaclass(abc.ABCMeta)   # Remove that line when discarding python2 support.
+# Python3 way: class MISPObjectGenerator(metaclass=abc.ABCMeta):
+class AbstractMISPObjectGenerator(MISPObject):
+
+    @abc.abstractmethod
+    def generate_attributes(self):
+        """Contains the logic where all the values of the object are gathered"""
+        pass

pymisp/exceptions.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+
 class PyMISPError(Exception):
     def __init__(self, message):
         super(PyMISPError, self).__init__(message)
@@ -29,3 +30,16 @@ class NoURL(PyMISPError):
 
 class NoKey(PyMISPError):
     pass
+
+
+class MISPObjectException(PyMISPError):
+    pass
+
+
+class InvalidMISPObject(MISPObjectException):
+    """Exception raised when an object doesn't respect the contrains in the definition"""
+    pass
+
+class UnknownMISPObjectTemplate(MISPObjectException):
+    """Exception raised when the template is unknown"""
+    pass

pymisp/mispevent.py

@@ -11,7 +11,6 @@ import base64
 from io import BytesIO
 from zipfile import ZipFile
 import hashlib
-from .abstract import AbstractMISP
 
 try:
     from dateutil.parser import parse
@@ -38,6 +37,7 @@ except ImportError:
         has_pyme = False
 
 from .exceptions import PyMISPError, NewEventError, NewAttributeError
+from .defaultobjects import MISPObject
 
 # Least dirty way to support python 2 and 3
 try:
@@ -285,7 +285,7 @@ def _int_to_str(d):
     return d
 
 
-class MISPEvent(AbstractMISP):
+class MISPEvent(object):
 
     def __init__(self, describe_types=None, strict_validation=False):
         self.ressources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data')
@@ -338,6 +338,7 @@ class MISPEvent(AbstractMISP):
         self.RelatedEvent = []
         self.Tag = []
         self.Galaxy = None
+        self.Object = None
 
     def _serialize(self):
         return '{date}{threat_level_id}{info}{uuid}{analysis}{timestamp}'.format(
@@ -510,6 +511,12 @@ class MISPEvent(AbstractMISP):
             self.sig = kwargs['sig']
         if kwargs.get('global_sig'):
             self.global_sig = kwargs['global_sig']
+        if kwargs.get('Object'):
+            self.Object = []
+            for obj in kwargs['Object']:
+                tmp_object = MISPObject(obj['name'])
+                tmp_object.from_dict(**obj)
+                self.Object.append(tmp_object)
 
     def _json(self):
         # DEPTECATED

pymisp/tools/__init__.py

@@ -1,5 +1,4 @@
 from .neo4j import Neo4j  # noqa
-from .objectgenerator import MISPObjectGenerator, MISPObjectException, InvalidMISPObject  # noqa
 from .fileobject import FileObject  # noqa
 from .peobject import PEObject, PESectionObject  # noqa
 from .elfobject import ELFObject, ELFSectionObject  # noqa

pymisp/tools/create_misp_object.py

@@ -1,7 +1,8 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-from pymisp.tools import FileObject, PEObject, ELFObject, MachOObject, MISPObjectException
+from pymisp.tools import FileObject, PEObject, ELFObject, MachOObject
+from pymisp.exceptions import MISPObjectException
 import warnings
 
 try:

pymisp/tools/elfobject.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-from pymisp.tools import MISPObjectGenerator
+from pymisp.defaultobjects import AbstractMISPObjectGenerator
 from io import BytesIO
 from hashlib import md5, sha1, sha256, sha512
 import warnings
@@ -20,7 +20,7 @@ except ImportError:
     HAS_PYDEEP = False
 
 
-class ELFObject(MISPObjectGenerator):
+class ELFObject(AbstractMISPObjectGenerator):
 
     def __init__(self, parsed=None, filepath=None, pseudofile=None):
         if not HAS_PYDEEP:
@@ -49,10 +49,10 @@ class ELFObject(MISPObjectGenerator):
 
     def generate_attributes(self):
         # General information
-        self._create_attribute('type', value=str(self.elf.header.file_type).split('.')[1])
+        self.add_attribute('type', value=str(self.elf.header.file_type).split('.')[1])
-        self._create_attribute('entrypoint-address', value=self.elf.entrypoint)
+        self.add_attribute('entrypoint-address', value=self.elf.entrypoint)
-        self._create_attribute('arch', value=str(self.elf.header.machine_type).split('.')[1])
+        self.add_attribute('arch', value=str(self.elf.header.machine_type).split('.')[1])
-        self._create_attribute('os_abi', value=str(self.elf.header.identity_os_abi).split('.')[1])
+        self.add_attribute('os_abi', value=str(self.elf.header.identity_os_abi).split('.')[1])
         # Sections
         self.sections = []
         if self.elf.sections:
@@ -62,10 +62,10 @@ class ELFObject(MISPObjectGenerator):
                 self.add_reference(s.uuid, 'included-in', 'Section {} of ELF'.format(pos))
                 pos += 1
                 self.sections.append(s)
-        self._create_attribute('number-sections', value=len(self.sections))
+        self.add_attribute('number-sections', value=len(self.sections))
 
 
-class ELFSectionObject(MISPObjectGenerator):
+class ELFSectionObject(AbstractMISPObjectGenerator):
 
     def __init__(self, section):
         # Python3 way
@@ -76,16 +76,16 @@ class ELFSectionObject(MISPObjectGenerator):
         self.generate_attributes()
 
     def generate_attributes(self):
-        self._create_attribute('name', value=self.section.name)
+        self.add_attribute('name', value=self.section.name)
-        self._create_attribute('type', value=str(self.section.type).split('.')[1])
+        self.add_attribute('type', value=str(self.section.type).split('.')[1])
         for flag in self.section.flags_list:
-            self._create_attribute('flag', value=str(flag).split('.')[1])
+            self.add_attribute('flag', value=str(flag).split('.')[1])
-        size = self._create_attribute('size-in-bytes', value=self.section.size)
+        size = self.add_attribute('size-in-bytes', value=self.section.size)
         if int(size.value) > 0:
-            self._create_attribute('entropy', value=self.section.entropy)
+            self.add_attribute('entropy', value=self.section.entropy)
-            self._create_attribute('md5', value=md5(self.data).hexdigest())
+            self.add_attribute('md5', value=md5(self.data).hexdigest())
-            self._create_attribute('sha1', value=sha1(self.data).hexdigest())
+            self.add_attribute('sha1', value=sha1(self.data).hexdigest())
-            self._create_attribute('sha256', value=sha256(self.data).hexdigest())
+            self.add_attribute('sha256', value=sha256(self.data).hexdigest())
-            self._create_attribute('sha512', value=sha512(self.data).hexdigest())
+            self.add_attribute('sha512', value=sha512(self.data).hexdigest())
             if HAS_PYDEEP:
-                self._create_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode())
+                self.add_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode())

pymisp/tools/fileobject.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-from pymisp.tools import MISPObjectGenerator
+from pymisp.defaultobjects import AbstractMISPObjectGenerator
 import os
 from io import BytesIO
 from hashlib import md5, sha1, sha256, sha512
@@ -22,7 +22,7 @@ except ImportError:
     HAS_MAGIC = False
 
 
-class FileObject(MISPObjectGenerator):
+class FileObject(AbstractMISPObjectGenerator):
 
     def __init__(self, filepath=None, pseudofile=None, filename=None):
         if not HAS_PYDEEP:
@@ -48,19 +48,19 @@ class FileObject(MISPObjectGenerator):
         self.generate_attributes()
 
     def generate_attributes(self):
-        self._create_attribute('filename', value=self.filename)
+        self.add_attribute('filename', value=self.filename)
-        size = self._create_attribute('size-in-bytes', value=len(self.data))
+        size = self.add_attribute('size-in-bytes', value=len(self.data))
         if int(size.value) > 0:
-            self._create_attribute('entropy', value=self.__entropy_H(self.data))
+            self.add_attribute('entropy', value=self.__entropy_H(self.data))
-            self._create_attribute('md5', value=md5(self.data).hexdigest())
+            self.add_attribute('md5', value=md5(self.data).hexdigest())
-            self._create_attribute('sha1', value=sha1(self.data).hexdigest())
+            self.add_attribute('sha1', value=sha1(self.data).hexdigest())
-            self._create_attribute('sha256', value=sha256(self.data).hexdigest())
+            self.add_attribute('sha256', value=sha256(self.data).hexdigest())
-            self._create_attribute('sha512', value=sha512(self.data).hexdigest())
+            self.add_attribute('sha512', value=sha512(self.data).hexdigest())
-            self._create_attribute('malware-sample', value=self.filename, data=self.pseudofile)
+            self.add_attribute('malware-sample', value=self.filename, data=self.pseudofile)
             if HAS_MAGIC:
-                self._create_attribute('mimetype', value=magic.from_buffer(self.data))
+                self.add_attribute('mimetype', value=magic.from_buffer(self.data))
             if HAS_PYDEEP:
-                self._create_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode())
+                self.add_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode())
 
     def __entropy_H(self, data):
         """Calculate the entropy of a chunk of data."""

pymisp/tools/machoobject.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-from pymisp.tools import MISPObjectGenerator
+from pymisp.defaultobjects import AbstractMISPObjectGenerator
 from io import BytesIO
 from hashlib import md5, sha1, sha256, sha512
 import warnings
@@ -20,7 +20,7 @@ except ImportError:
     HAS_PYDEEP = False
 
 
-class MachOObject(MISPObjectGenerator):
+class MachOObject(AbstractMISPObjectGenerator):
 
     def __init__(self, parsed=None, filepath=None, pseudofile=None):
         if not HAS_PYDEEP:
@@ -48,11 +48,11 @@ class MachOObject(MISPObjectGenerator):
         self.generate_attributes()
 
     def generate_attributes(self):
-        self._create_attribute('type', value=str(self.macho.header.file_type).split('.')[1])
+        self.add_attribute('type', value=str(self.macho.header.file_type).split('.')[1])
-        self._create_attribute('name', value=self.macho.name)
+        self.add_attribute('name', value=self.macho.name)
         # General information
         if self.macho.has_entrypoint:
-            self._create_attribute('entrypoint-address', value=self.macho.entrypoint)
+            self.add_attribute('entrypoint-address', value=self.macho.entrypoint)
         # Sections
         self.sections = []
         if self.macho.sections:
@@ -62,10 +62,10 @@ class MachOObject(MISPObjectGenerator):
                 self.add_reference(s.uuid, 'included-in', 'Section {} of MachO'.format(pos))
                 pos += 1
                 self.sections.append(s)
-        self._create_attribute('number-sections', value=len(self.sections))
+        self.add_attribute('number-sections', value=len(self.sections))
 
 
-class MachOSectionObject(MISPObjectGenerator):
+class MachOSectionObject(AbstractMISPObjectGenerator):
 
     def __init__(self, section):
         # Python3 way
@@ -76,13 +76,13 @@ class MachOSectionObject(MISPObjectGenerator):
         self.generate_attributes()
 
     def generate_attributes(self):
-        self._create_attribute('name', value=self.section.name)
+        self.add_attribute('name', value=self.section.name)
-        size = self._create_attribute('size-in-bytes', value=self.section.size)
+        size = self.add_attribute('size-in-bytes', value=self.section.size)
         if int(size.value) > 0:
-            self._create_attribute('entropy', value=self.section.entropy)
+            self.add_attribute('entropy', value=self.section.entropy)
-            self._create_attribute('md5', value=md5(self.data).hexdigest())
+            self.add_attribute('md5', value=md5(self.data).hexdigest())
-            self._create_attribute('sha1', value=sha1(self.data).hexdigest())
+            self.add_attribute('sha1', value=sha1(self.data).hexdigest())
-            self._create_attribute('sha256', value=sha256(self.data).hexdigest())
+            self.add_attribute('sha256', value=sha256(self.data).hexdigest())
-            self._create_attribute('sha512', value=sha512(self.data).hexdigest())
+            self.add_attribute('sha512', value=sha512(self.data).hexdigest())
             if HAS_PYDEEP:
-                self._create_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode())
+                self.add_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode())

pymisp/tools/objectgenerator.py

@@ -1,142 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from collections import Counter
-from pymisp import MISPEvent, MISPAttribute, AbstractMISP
-import os
-import json
-import uuid
-import abc
-import sys
-import six  # Remove that import when discarding python2 support.
-
-
-class MISPObjectException(Exception):
-    pass
-
-
-class InvalidMISPObject(MISPObjectException):
-    """Exception raised when an object doesn't contains the required field(s)"""
-    pass
-
-
-if six.PY2:
-    import warnings
-    warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5")
-
-
-class MISPObjectReference(AbstractMISP):
-
-    attributes = ['source_uuid', 'destination_uuid', 'relationship_type', 'comment']
-
-    def __init__(self, source_uuid, destination_uuid, relationship_type, comment=None):
-        self.source_uuid = source_uuid
-        self.destination_uuid = destination_uuid
-        self.relationship_type = relationship_type
-        self.comment = comment
-
-
-class MISPObjectAttribute(AbstractMISP):
-
-    # This list is very limited and hardcoded to fit the current needs (file/pe/pesection creation): MISPAttriute will follow the
-    # same spec and just add one attribute: object_relation
-    attributes = ['object_relation', 'value', 'type', 'category', 'disable_correlation', 'to_ids',
-                  'data', 'encrypt', 'distribution', 'comment']
-
-    def __init__(self, definition, object_relation, value, **kwargs):
-        self.object_relation = object_relation
-        self.value = value
-        # Initialize the new MISPAttribute
-        # Get the misp attribute type from the definition
-        self.type = kwargs.pop('type', None)
-        if self.type is None:
-            self.type = definition['misp-attribute']
-        self.disable_correlation = kwargs.pop('disable_correlation', None)
-        if self.disable_correlation is None:
-            # The correlation can be disabled by default in the object definition.
-            # Use this value if it isn't overloaded by the object
-            self.disable_correlation = definition.get('disable_correlation')
-        self.to_ids = kwargs.pop('to_ids', None)
-        if self.to_ids is None:
-            # Same for the to_ids flag
-            self.to_ids = definition.get('to_ids')
-        # Initialise rest of the values
-        for k, v in kwargs.items():
-            self[k] = v
-        # FIXME: dirty hack until all the classes are ported to the new format but we get the default values
-        temp_attribute = MISPAttribute()
-        temp_attribute.set_all_values(**self)
-        # Update default values
-        self.from_dict(**temp_attribute.to_dict())
-
-
-class MISPObjectGenerator(AbstractMISP):
-
-    attributes = ['name', 'meta-category', 'uuid', 'description', 'version', 'Attribute']
-
-    def __init__(self, template_dir):
-        """This class is used to fill a new MISP object with the default values defined in the object template
-            * template is the path to the template within the misp-object repository
-            * misp_objects_path is the path to the misp-object repository
-        """
-        self.misp_objects_path = os.path.join(
-            os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)),
-            'data', 'misp-objects', 'objects')
-        with open(os.path.join(self.misp_objects_path, template_dir, 'definition.json'), 'r') as f:
-            self.definition = json.load(f)
-        self.misp_event = MISPEvent()
-        self.name = self.definition['name']
-        setattr(self, 'meta-category', self.definition['meta-category'])
-        self.template_uuid = self.definition['uuid']
-        self.description = self.definition['description']
-        self.version = self.definition['version']
-        self.uuid = str(uuid.uuid4())
-        self.Attribute = []
-        self.references = []
-
-    def _create_attribute(self, object_type, **value):
-        if value.get('value') is None:
-            return None
-        attribute = MISPObjectAttribute(self.definition['attributes'][object_type], object_type, **value)
-        self.Attribute.append(attribute)
-        return attribute
-
-    def to_dict(self, strict=True):
-        if strict:
-            self._validate()
-        return super(MISPObjectGenerator, self).to_dict()
-
-    def to_json(self, strict=True):
-        if strict:
-            self._validate()
-        return super(MISPObjectGenerator, self).to_json()
-
-    def _validate(self):
-        """Make sure the object we're creating has the required fields"""
-        all_object_relations = []
-        for a in self.Attribute:
-            all_object_relations.append(a.object_relation)
-        count_relations = dict(Counter(all_object_relations))
-        for key, counter in count_relations.items():
-            if counter == 1:
-                continue
-            if not self.definition['attributes'][key].get('multiple'):
-                raise InvalidMISPObject('Multiple occurrences of {} is not allowed'.format(key))
-        all_attribute_names = set(count_relations.keys())
-        if self.definition.get('requiredOneOf'):
-            if not set(self.definition['requiredOneOf']) & all_attribute_names:
-                raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self.definition['requiredOneOf'])))
-        if self.definition.get('required'):
-            for r in self.definition.get('required'):
-                if r not in all_attribute_names:
-                    raise InvalidMISPObject('{} is required'.format(r))
-        return True
-
-    def add_reference(self, destination_uuid, relationship_type, comment=None):
-        """Add a link (uuid) to an other object"""
-        self.references.append(MISPObjectReference(self.uuid, destination_uuid, relationship_type, comment))
-
-    @abc.abstractmethod
-    def generate_attributes(self):
-        """Contains the logic where all the values of the object are gathered"""
-        pass

pymisp/tools/peobject.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-from pymisp.tools import MISPObjectGenerator
+from pymisp.defaultobjects import AbstractMISPObjectGenerator
 from io import BytesIO
 from hashlib import md5, sha1, sha256, sha512
 from datetime import datetime
@@ -21,7 +21,7 @@ except ImportError:
     HAS_PYDEEP = False
 
 
-class PEObject(MISPObjectGenerator):
+class PEObject(AbstractMISPObjectGenerator):
 
     def __init__(self, parsed=None, filepath=None, pseudofile=None):
         if not HAS_PYDEEP:
@@ -74,10 +74,10 @@ class PEObject(MISPObjectGenerator):
             return 'unknown'
 
     def generate_attributes(self):
-        self._create_attribute('type', value=self._get_pe_type())
+        self.add_attribute('type', value=self._get_pe_type())
         # General information
-        self._create_attribute('entrypoint-address', value=self.pe.entrypoint)
+        self.add_attribute('entrypoint-address', value=self.pe.entrypoint)
-        self._create_attribute('compilation-timestamp', value=datetime.utcfromtimestamp(self.pe.header.time_date_stamps).isoformat())
+        self.add_attribute('compilation-timestamp', value=datetime.utcfromtimestamp(self.pe.header.time_date_stamps).isoformat())
         # self.imphash = self.pe.get_imphash()
         try:
             if (self.pe.has_resources and
@@ -85,15 +85,15 @@ class PEObject(MISPObjectGenerator):
                     self.pe.resources_manager.version.has_string_file_info and
                     self.pe.resources_manager.version.string_file_info.langcode_items):
                 fileinfo = dict(self.pe.resources_manager.version.string_file_info.langcode_items[0].items.items())
-                self._create_attribute('original-filename', value=fileinfo.get('OriginalFilename'))
+                self.add_attribute('original-filename', value=fileinfo.get('OriginalFilename'))
-                self._create_attribute('internal-filename', value=fileinfo.get('InternalName'))
+                self.add_attribute('internal-filename', value=fileinfo.get('InternalName'))
-                self._create_attribute('file-description', value=fileinfo.get('FileDescription'))
+                self.add_attribute('file-description', value=fileinfo.get('FileDescription'))
-                self._create_attribute('file-version', value=fileinfo.get('FileVersion'))
+                self.add_attribute('file-version', value=fileinfo.get('FileVersion'))
-                self._create_attribute('lang-id', value=self.pe.resources_manager.version.string_file_info.langcode_items[0].key)
+                self.add_attribute('lang-id', value=self.pe.resources_manager.version.string_file_info.langcode_items[0].key)
-                self._create_attribute('product-name', value=fileinfo.get('ProductName'))
+                self.add_attribute('product-name', value=fileinfo.get('ProductName'))
-                self._create_attribute('product-version', value=fileinfo.get('ProductVersion'))
+                self.add_attribute('product-version', value=fileinfo.get('ProductVersion'))
-                self._create_attribute('company-name', value=fileinfo.get('CompanyName'))
+                self.add_attribute('company-name', value=fileinfo.get('CompanyName'))
-                self._create_attribute('legal-copyright', value=fileinfo.get('LegalCopyright'))
+                self.add_attribute('legal-copyright', value=fileinfo.get('LegalCopyright'))
         except lief.read_out_of_bound:
             # The file is corrupted
             pass
@@ -106,14 +106,14 @@ class PEObject(MISPObjectGenerator):
                 self.add_reference(s.uuid, 'included-in', 'Section {} of PE'.format(pos))
                 if ((self.pe.entrypoint >= section.virtual_address) and
                         (self.pe.entrypoint < (section.virtual_address + section.virtual_size))):
-                    self._create_attribute('entrypoint-section|position', value='{}|{}'.format(section.name, pos))
+                    self.add_attribute('entrypoint-section|position', value='{}|{}'.format(section.name, pos))
                 pos += 1
                 self.sections.append(s)
-        self._create_attribute('number-sections', value=len(self.sections))
+        self.add_attribute('number-sections', value=len(self.sections))
         # TODO: TLSSection / DIRECTORY_ENTRY_TLS
 
 
-class PESectionObject(MISPObjectGenerator):
+class PESectionObject(AbstractMISPObjectGenerator):
 
     def __init__(self, section):
         # Python3 way
@@ -124,13 +124,13 @@ class PESectionObject(MISPObjectGenerator):
         self.generate_attributes()
 
     def generate_attributes(self):
-        self._create_attribute('name', value=self.section.name)
+        self.add_attribute('name', value=self.section.name)
-        size = self._create_attribute('size-in-bytes', value=self.section.size)
+        size = self.add_attribute('size-in-bytes', value=self.section.size)
         if int(size.value) > 0:
-            self._create_attribute('entropy', value=self.section.entropy)
+            self.add_attribute('entropy', value=self.section.entropy)
-            self._create_attribute('md5', value=md5(self.data).hexdigest())
+            self.add_attribute('md5', value=md5(self.data).hexdigest())
-            self._create_attribute('sha1', value=sha1(self.data).hexdigest())
+            self.add_attribute('sha1', value=sha1(self.data).hexdigest())
-            self._create_attribute('sha256', value=sha256(self.data).hexdigest())
+            self.add_attribute('sha256', value=sha256(self.data).hexdigest())
-            self._create_attribute('sha512', value=sha512(self.data).hexdigest())
+            self.add_attribute('sha512', value=sha512(self.data).hexdigest())
             if HAS_PYDEEP:
-                self._create_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode())
+                self.add_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode())

tests/test_offline.py

@@ -230,18 +230,18 @@ class TestOffline(unittest.TestCase):
         if seos:
             for s in seos:
                 to_return['objects'].append(s)
-                if s.references:
+                if s.ObjectReference:
-                    to_return['references'] += s.references
+                    to_return['references'] += s.ObjectReference
 
         if peo:
             to_return['objects'].append(peo)
-            if peo.references:
+            if peo.ObjectReference:
-                to_return['references'] += peo.references
+                to_return['references'] += peo.ObjectReference
 
         if fo:
             to_return['objects'].append(fo)
-            if fo.references:
+            if fo.ObjectReference:
-                to_return['references'] += fo.references
+                to_return['references'] += fo.ObjectReference
         return json.dumps(to_return, cls=MISPEncode)
 
     def test_objects(self, m):