Update file/pe/pe-sections objects creator.

examples/add_file_object.py

@@ -7,7 +7,7 @@ import traceback
 from keys import misp_url, misp_key, misp_verifycert
 import glob
 import argparse
-
+import json
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Extract indicators out of binaries and add MISP objects to a MISP instance.')
@@ -18,31 +18,27 @@ if __name__ == '__main__':
     pymisp = PyMISP(misp_url, misp_key, misp_verifycert)
 
     for f in glob.glob(args.path):
-        print('\n', f)
         try:
             fo, peo, seos = make_binary_objects(f)
         except Exception as e:
             traceback.print_exc()
-            continue
+
-        if fo:
-            template_id = pymisp.get_object_template_id(fo['name'])
-            try:
-                response = pymisp.add_object(args.event, template_id, fo)
-                print(response)
-            except Exception as e:
-                traceback.print_exc()
-                continue
-        continue
-        if peo:
-            template_id = pymisp.get_object_template_id(peo['name'])
-            print(template_id)
-            r = pymisp.add_object(args.event, template_id, peo)
-            print(r)
-        continue
         if seos:
             for s in seos:
-                print(s)
+                obj, refs = s
-                template_id = pymisp.get_object_template_id(s['name'])
+                template_id = pymisp.get_object_template_id(obj['name'])
-                r = pymisp.add_object(args.event, template_id, s)
+                r = pymisp.add_object(args.event, template_id, obj)
-                print(r)
+
-                break
+        if peo:
+            obj, refs = peo
+            template_id = pymisp.get_object_template_id(obj['name'])
+            r = pymisp.add_object(args.event, template_id, obj)
+            for ref in refs:
+                r = pymisp.add_object_reference(obj['uuid'], ref)
+
+        if fo:
+            obj, refs = fo
+            template_id = pymisp.get_object_template_id(obj['name'])
+            response = pymisp.add_object(args.event, template_id, obj)
+            for ref in refs:
+                r = pymisp.add_object_reference(obj['uuid'], ref)

pymisp/__init__.py

@@ -2,6 +2,7 @@ __version__ = '2.4.77'
 
 from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey
 from .api import PyMISP
+from .abstract import AbstractMISP
 from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull
 from .tools import Neo4j
 from .tools import stix

pymisp/abstract.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import six  # Remove that import when discarding python2 support.
+import abc
+import json
+import collections
+
+
+@six.add_metaclass(abc.ABCMeta)   # Remove that line when discarding python2 support.
+# Python3 way: class MISPObjectGenerator(metaclass=abc.ABCMeta):
+class AbstractMISP(collections.MutableMapping):
+
+    attributes = None
+
+    def __init__(self):
+        """Initialize the list of class-level attributes to set in the JSON dump"""
+        # The attribute names will be set automatically by the schemas when we will have them.
+        if self.attributes is None:
+            raise NotImplementedError('{} must define attributes'.format(type(self).__name__))
+        self.attributes = sorted(self.attributes)
+
+    def __check_dict_key(self, key):
+        if key not in self.attributes:
+            raise Exception('{} not a valid key in {}. Alowed keys: {}'.format(
+                key, type(self).__name__, ', '.join(self.attributes)))
+        return True
+
+    def from_dict(self, **kwargs):
+        for attribute in self.attributes:
+            val = kwargs.pop(attribute, None)
+            if val is None:
+                continue
+            setattr(self, attribute, val)
+        if kwargs:
+            raise Exception('Unused parameter(s): {}'.format(', '.join(kwargs.keys())))
+
+    def from_json(self, json_string):
+        """Load a JSON string"""
+        self.from_dict(json.loads(json_string))
+
+    def to_dict(self):
+        to_return = {}
+        for attribute in self.attributes:
+            val = getattr(self, attribute, None)
+            if val is None:
+                continue
+            to_return[attribute] = val
+        return to_return
+
+    def to_json(self):
+        return json.dumps(self.to_dict(), indent=4, sort_keys=True)
+
+    def __getitem__(self, key):
+        if self.__check_dict_key(key):
+            return getattr(self, key)
+
+    def __setitem__(self, key, value):
+        if self.__check_dict_key(key):
+            setattr(self, key, value)
+
+    def __delitem__(self, key):
+        if self.__check_dict_key(key):
+            delattr(self, key)
+
+    def __iter__(self):
+        return iter(self.to_dict())
+
+    def __len__(self):
+        return len(self.to_dict())

pymisp/api.py

@@ -1588,6 +1588,12 @@ class PyMISP(object):
         response = session.post(url, data=json.dumps(misp_object))
         return self._check_response(response)
 
+    def add_object_reference(self, parent_uuid, misp_object_reference):
+        session = self.__prepare_session()
+        url = urljoin(self.root_url, 'object_references/add/{}'.format(parent_uuid))
+        response = session.post(url, data=json.dumps(misp_object_reference))
+        return self._check_response(response)
+
     def get_object_templates_list(self):
         session = self.__prepare_session()
         url = urljoin(self.root_url, 'objectTemplates')

pymisp/data/misp-objects

@@ -1 +1 @@
-Subproject commit ca24684e2f49bcfbd886212ff003472716c26de9
+Subproject commit 96d7aeb0729428a43f38f45b6c00a60b9fdba2b6

pymisp/mispevent.py

@@ -11,6 +11,7 @@ import base64
 from io import BytesIO
 from zipfile import ZipFile
 import hashlib
+from .abstract import AbstractMISP
 
 try:
     from dateutil.parser import parse
@@ -198,8 +199,7 @@ class MISPAttribute(object):
             self.malware_filename = self.value
         m = hashlib.md5()
         m.update(self.data.getvalue())
-        md5 = m.hexdigest()
+        self.value = self.malware_filename
-        self.value = '{}|{}'.format(self.malware_filename, md5)
         self.malware_binary = self.data
         self.encrypt = True
 
@@ -280,7 +280,7 @@ def _int_to_str(d):
     return d
 
 
-class MISPEvent(object):
+class MISPEvent(AbstractMISP):
 
     def __init__(self, describe_types=None, strict_validation=False):
         self.ressources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data')

pymisp/tools/create_misp_object.py

@@ -16,7 +16,7 @@ class FileTypeNotImplemented(MISPObjectException):
 
 def make_pe_objects(lief_parsed, misp_file):
     misp_pe = PEObject(parsed=lief_parsed)
-    misp_file.add_link(misp_pe.uuid, 'PE indicators')
+    misp_file.add_reference(misp_pe.uuid, 'included-in', 'PE indicators')
     file_object = misp_file.dump()
     pe_object = misp_pe.dump()
     pe_sections = []

pymisp/tools/fileobject.py

@@ -45,15 +45,17 @@ class FileObject(MISPObjectGenerator):
         self.generate_attributes()
 
     def generate_attributes(self):
-        self.size = len(self.data)
+        self._create_attribute('filename', value=self.filename)
-        if self.size > 0:
+        self._create_attribute('size-in-bytes', value=len(self.data))
-            self.entropy = self.__entropy_H(self.data)
+        if getattr(self, 'size-in-bytes').value > 0:
-            self.md5 = md5(self.data).hexdigest()
+            self._create_attribute('entropy', value=self.__entropy_H(self.data))
-            self.sha1 = sha1(self.data).hexdigest()
+            self._create_attribute('md5', value=md5(self.data).hexdigest())
-            self.sha256 = sha256(self.data).hexdigest()
+            self._create_attribute('sha1', value=sha1(self.data).hexdigest())
-            self.sha512 = sha512(self.data).hexdigest()
+            self._create_attribute('sha256', value=sha256(self.data).hexdigest())
-            self.filetype = magic.from_buffer(self.data)
+            self._create_attribute('sha512', value=sha512(self.data).hexdigest())
-            self.ssdeep = pydeep.hash_buf(self.data).decode()
+            self._create_attribute('mimetype', value=magic.from_buffer(self.data))
+            self._create_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode())
+            self._create_attribute('malware-sample', value=self.filename.value, data=self.pseudofile)
 
     def __entropy_H(self, data):
         """Calculate the entropy of a chunk of data."""
@@ -70,23 +72,3 @@ class FileObject(MISPObjectGenerator):
             entropy -= p_x * math.log(p_x, 2)
 
         return entropy
-
-    def dump(self):
-        file_object = {}
-        file_object['filename'] = {'value': self.filename}
-        file_object['size-in-bytes'] = {'value': self.size}
-        if self.size > 0:
-            file_object['entropy'] = {'value': self.entropy}
-            file_object['ssdeep'] = {'value': self.ssdeep}
-            file_object['sha512'] = {'value': self.sha512}
-            file_object['md5'] = {'value': self.md5}
-            file_object['sha1'] = {'value': self.sha1}
-            file_object['sha256'] = {'value': self.sha256}
-            file_object['malware-sample'] = {'value': '{}|{}'.format(self.filename, self.md5), 'data': self.pseudofile}
-            # file_object['authentihash'] = self.
-            # file_object['sha-224'] = self.
-            # file_object['sha-384'] = self.
-            # file_object['sha512/224'] = self.
-            # file_object['sha512/256'] = self.
-            # file_object['tlsh'] = self.
-        return self._fill_object(file_object)

pymisp/tools/objectgenerator.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-from pymisp import MISPEvent, MISPAttribute
+from pymisp import MISPEvent, MISPAttribute, AbstractMISP
 import os
 import json
 import uuid
@@ -24,9 +24,17 @@ if six.PY2:
     warnings.warn("You're using python 2, it is strongly recommended to use python >=3.4")
 
 
-@six.add_metaclass(abc.ABCMeta)   # Remove that line when discarding python2 support.
+class MISPObjectReference(AbstractMISP):
-# Python3 way: class MISPObjectGenerator(metaclass=abc.ABCMeta):
+
-class MISPObjectGenerator():
+    attributes = ['uuid', 'relationship_type', 'comment']
+
+    def __init__(self, uuid, relationship_type, comment=None):
+        self['uuid'] = uuid
+        self['relationship_type'] = relationship_type
+        self['comment'] = comment
+
+
+class MISPObjectGenerator(AbstractMISP):
 
     def __init__(self, template_dir):
         """This class is used to fill a new MISP object with the default values defined in the object template
@@ -38,46 +46,46 @@ class MISPObjectGenerator():
             'data', 'misp-objects', 'objects')
         with open(os.path.join(self.misp_objects_path, template_dir, 'definition.json'), 'r') as f:
             self.definition = json.load(f)
+        self.attributes = self.definition['attributes'].keys()
         self.misp_event = MISPEvent()
         self.uuid = str(uuid.uuid4())
-        self.links = []
+        self.references = []
 
-    def _fill_object(self, values, strict=True):
+    def _create_attribute(self, object_type, **value):
+        if value.get('value') is None:
+            return None
+        # Initialize the new MISPAttribute
+        # Get the misp attribute type from the definition
+        value['type'] = self.definition['attributes'][object_type]['misp-attribute']
+        if value.get('disable_correlation') is None:
+            # The correlation can be disabled by default in the object definition.
+            # Use this value if it isn't overloaded by the object
+            value['disable_correlation'] = self.definition['attributes'][object_type].get('disable_correlation')
+        if value.get('to_ids') is None:
+            # Same for the to_ids flag
+            value['to_ids'] = self.definition['attributes'][object_type].get('to_ids')
+        # Set all the values in the MISP attribute
+        attribute = MISPAttribute(self.misp_event.describe_types)
+        attribute.set_all_values(**value)
+        self[object_type] = attribute
+
+    def dump(self, strict=True):
         """Create a new object with the values gathered by the sub-class, use the default values from the template if needed"""
         if strict:
-            self._validate(values)
+            self._validate()
         # Create an empty object based om the object definition
         new_object = self.__new_empty_object(self.definition)
-        if self.links:
+        for object_type, attribute in self.items():
-            # Set the links to other objects
-            new_object["ObjectReference"] = []
-            for link in self.links:
-                uuid, comment = link
-                new_object['ObjectReference'].append({'referenced_object_uuid': uuid, 'comment': comment})
-        for object_type, value in values.items():
             # Add all the values as MISPAttributes to the current object
-            if value.get('value') is None:
+            if attribute.value is None:
                 continue
-            # Initialize the new MISPAttribute
-            attribute = MISPAttribute(self.misp_event.describe_types)
-            # Get the misp attribute type from the definition
-            value['type'] = self.definition['attributes'][object_type]['misp-attribute']
-            if value.get('disable_correlation') is None:
-                # The correlation can be disabled by default in the object definition.
-                # Use this value if it isn't overloaded by the object
-                value['disable_correlation'] = self.definition['attributes'][object_type].get('disable_correlation')
-            if value.get('to_ids') is None:
-                # Same for the to_ids flag
-                value['to_ids'] = self.definition['attributes'][object_type].get('to_ids')
-            # Set all the values in the MISP attribute
-            attribute.set_all_values(**value)
             # Finalize the actual MISP Object
-            new_object['Attribute'].append({'type': object_type, 'Attribute': attribute._json()})
+            new_object['Attribute'].append({'object_relation': object_type, **attribute._json()})
-        return new_object
+        return new_object, [r.to_dict() for r in self.references]
 
-    def _validate(self, dump):
+    def _validate(self):
         """Make sure the object we're creating has the required fields"""
-        all_attribute_names = set(dump.keys())
+        all_attribute_names = set(self.keys())
         if self.definition.get('requiredOneOf'):
             if not set(self.definition['requiredOneOf']) & all_attribute_names:
                 raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self.definition['requiredOneOf'])))
@@ -87,9 +95,9 @@ class MISPObjectGenerator():
                     raise InvalidMISPObject('{} is required is required'.format(r))
         return True
 
-    def add_link(self, uuid, comment=None):
+    def add_reference(self, uuid, relationship_type, comment=None):
         """Add a link (uuid) to an other object"""
-        self.links.append((uuid, comment))
+        self.references.append(MISPObjectReference(uuid, relationship_type, comment))
 
     def __new_empty_object(self, object_definiton):
         """Create a new empty object out of the template"""
@@ -101,10 +109,3 @@ class MISPObjectGenerator():
     def generate_attributes(self):
         """Contains the logic where all the values of the object are gathered"""
         pass
-
-    @abc.abstractmethod
-    def dump(self):
-        """This method normalize the attributes to add to the object.
-        It returns an python dictionary where the key is the type defined in the object,
-        and the value the value of the MISP Attribute"""
-        pass

pymisp/tools/peobject.py

@@ -60,18 +60,21 @@ class PEObject(MISPObjectGenerator):
             return True
         return False
 
-    def generate_attributes(self):
+    def _get_pe_type(self):
         if self._is_dll():
-            self.pe_type = 'dll'
+            return 'dll'
         elif self._is_driver():
-            self.pe_type = 'driver'
+            return 'driver'
         elif self._is_exe():
-            self.pe_type = 'exe'
+            return 'exe'
         else:
-            self.pe_type = 'unknown'
+            return 'unknown'
+
+    def generate_attributes(self):
+        self._create_attribute('type', value=self._get_pe_type())
         # General information
-        self.entrypoint_address = self.pe.entrypoint
+        self._create_attribute('entrypoint-address', value=self.pe.entrypoint)
-        self.compilation_timestamp = datetime.utcfromtimestamp(self.pe.header.time_date_stamps).isoformat()
+        self._create_attribute('compilation-timestamp', value=datetime.utcfromtimestamp(self.pe.header.time_date_stamps).isoformat())
         # self.imphash = self.pe.get_imphash()
         try:
             if (self.pe.has_resources and
@@ -79,15 +82,15 @@ class PEObject(MISPObjectGenerator):
                     self.pe.resources_manager.version.has_string_file_info and
                     self.pe.resources_manager.version.string_file_info.langcode_items):
                 fileinfo = dict(self.pe.resources_manager.version.string_file_info.langcode_items[0].items.items())
-                self.original_filename = fileinfo.get('OriginalFilename')
+                self._create_attribute('original-filename', value=fileinfo.get('OriginalFilename'))
-                self.internal_filename = fileinfo.get('InternalName')
+                self._create_attribute('internal-filename', value=fileinfo.get('InternalName'))
-                self.file_description = fileinfo.get('FileDescription')
+                self._create_attribute('file-description', value=fileinfo.get('FileDescription'))
-                self.file_version = fileinfo.get('FileVersion')
+                self._create_attribute('file-version', value=fileinfo.get('FileVersion'))
-                self.lang_id = self.pe.resources_manager.version.string_file_info.langcode_items[0].key
+                self._create_attribute('lang-id', value=self.pe.resources_manager.version.string_file_info.langcode_items[0].key)
-                self.product_name = fileinfo.get('ProductName')
+                self._create_attribute('product-name', value=fileinfo.get('ProductName'))
-                self.product_version = fileinfo.get('ProductVersion')
+                self._create_attribute('product-version', value=fileinfo.get('ProductVersion'))
-                self.company_name = fileinfo.get('CompanyName')
+                self._create_attribute('company-name', value=fileinfo.get('CompanyName'))
-                self.legal_copyright = fileinfo.get('LegalCopyright')
+                self._create_attribute('legal-copyright', value=fileinfo.get('LegalCopyright'))
         except lief.read_out_of_bound:
             # The file is corrupted
             pass
@@ -97,46 +100,15 @@ class PEObject(MISPObjectGenerator):
             pos = 0
             for section in self.pe.sections:
                 s = PESectionObject(section)
-                self.add_link(s.uuid, 'Section {} of PE'.format(pos))
+                self.add_reference(s.uuid, 'included-in', 'Section {} of PE'.format(pos))
-                if ((self.entrypoint_address >= section.virtual_address) and
+                if ((self.pe.entrypoint >= section.virtual_address) and
-                        (self.entrypoint_address < (section.virtual_address + section.virtual_size))):
+                        (self.pe.entrypoint < (section.virtual_address + section.virtual_size))):
-                    self.entrypoint_section = (section.name, pos)  # Tuple: (section_name, position)
+                    self._create_attribute('entrypoint-section|position', value='{}|{}'.format(section.name, pos))
                 pos += 1
                 self.sections.append(s)
-        self.nb_sections = len(self.sections)
+        self._create_attribute('number-sections', value=len(self.sections))
         # TODO: TLSSection / DIRECTORY_ENTRY_TLS
 
-    def dump(self):
-        pe_object = {}
-        pe_object['type'] = {'value': self.pe_type}
-        if hasattr(self, 'imphash'):
-            pe_object['imphash'] = {'value': self.imphash}
-        if hasattr(self, 'original_filename'):
-            pe_object['original-filename'] = {'value': self.original_filename}
-        if hasattr(self, 'internal_filename'):
-            pe_object['internal-filename'] = {'value': self.internal_filename}
-        if hasattr(self, 'compilation_timestamp'):
-            pe_object['compilation-timestamp'] = {'value': self.compilation_timestamp}
-        if hasattr(self, 'entrypoint_section'):
-            pe_object['entrypoint-section|position'] = {'value': '{}|{}'.format(*self.entrypoint_section)}
-        if hasattr(self, 'entrypoint_address'):
-            pe_object['entrypoint-address'] = {'value': self.entrypoint_address}
-        if hasattr(self, 'file_description'):
-            pe_object['file-description'] = {'value': self.file_description}
-        if hasattr(self, 'file_version'):
-            pe_object['file-version'] = {'value': self.file_version}
-        if hasattr(self, 'lang_id'):
-            pe_object['lang-id'] = {'value': self.lang_id}
-        if hasattr(self, 'product_name'):
-            pe_object['product-name'] = {'value': self.product_name}
-        if hasattr(self, 'product_version'):
-            pe_object['product-version'] = {'value': self.product_version}
-        if hasattr(self, 'company_name'):
-            pe_object['company-name'] = {'value': self.company_name}
-        if hasattr(self, 'nb_sections'):
-            pe_object['number-sections'] = {'value': self.nb_sections}
-        return self._fill_object(pe_object)
-
 
 class PESectionObject(MISPObjectGenerator):
 
@@ -147,26 +119,13 @@ class PESectionObject(MISPObjectGenerator):
         self.generate_attributes()
 
     def generate_attributes(self):
-        self.name = self.section.name
+        self._create_attribute('name', value=self.section.name)
-        self.size = self.section.size
+        self._create_attribute('size-in-bytes', value=self.section.size)
-        if self.size > 0:
+        if getattr(self, 'size-in-bytes').value > 0:
-            self.entropy = self.section.entropy
+            self._create_attribute('entropy', value=self.section.entropy)
-            self.md5 = md5(self.data).hexdigest()
+            self._create_attribute('md5', value=md5(self.data).hexdigest())
-            self.sha1 = sha1(self.data).hexdigest()
+            self._create_attribute('sha1', value=sha1(self.data).hexdigest())
-            self.sha256 = sha256(self.data).hexdigest()
+            self._create_attribute('sha256', value=sha256(self.data).hexdigest())
-            self.sha512 = sha512(self.data).hexdigest()
+            self._create_attribute('sha512', value=sha512(self.data).hexdigest())
             if HAS_PYDEEP:
-                self.ssdeep = pydeep.hash_buf(self.data).decode()
+                self._create_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode())
-
-    def dump(self):
-        section = {}
-        section['name'] = {'value': self.name}
-        section['size-in-bytes'] = {'value': self.size}
-        if self.size > 0:
-            section['entropy'] = {'value': self.entropy}
-            section['md5'] = {'value': self.md5}
-            section['sha1'] = {'value': self.sha1}
-            section['sha256'] = {'value': self.sha256}
-            section['sha512'] = {'value': self.sha512}
-            section['ssdeep'] = {'value': self.ssdeep}
-        return self._fill_object(section)

setup.py

@@ -29,6 +29,8 @@ setup(
     test_suite="tests",
     install_requires=['requests', 'python-dateutil', 'jsonschema'],
     include_package_data=True,
-    package_data={'pymisp': ['data/*.json', 'data/misp-objects/schema.json',
+    package_data={'pymisp': ['data/*.json', 'data/misp-objects/schema_objects.json',
-                             'data/misp-objects/objects/*/definition.json']},
+                             'data/misp-objects/schema_relationships.json',
+                             'data/misp-objects/objects/*/definition.json',
+                             'data/misp-objects/relationships/definition.json']},
 )