Refactor all the things

Add script for MISP core, make everything generic.

examples/add_file_object.py

@@ -7,7 +7,6 @@ import traceback
 from keys import misp_url, misp_key, misp_verifycert
 import glob
 import argparse
-import json
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Extract indicators out of binaries and add MISP objects to a MISP instance.')
@@ -25,20 +24,17 @@ if __name__ == '__main__':
 
         if seos:
             for s in seos:
-                obj, refs = s
+                template_id = pymisp.get_object_template_id(s['name'])
-                template_id = pymisp.get_object_template_id(obj['name'])
+                r = pymisp.add_object(args.event, template_id, s)
-                r = pymisp.add_object(args.event, template_id, obj)
 
         if peo:
-            obj, refs = peo
+            template_id = pymisp.get_object_template_id(peo['name'])
-            template_id = pymisp.get_object_template_id(obj['name'])
+            r = pymisp.add_object(args.event, template_id, peo)
-            r = pymisp.add_object(args.event, template_id, obj)
+            for ref in peo.references:
-            for ref in refs:
+                r = pymisp.add_object_reference(ref)
-                r = pymisp.add_object_reference(obj['uuid'], ref)
 
         if fo:
-            obj, refs = fo
+            template_id = pymisp.get_object_template_id(fo['name'])
-            template_id = pymisp.get_object_template_id(obj['name'])
+            response = pymisp.add_object(args.event, template_id, fo)
-            response = pymisp.add_object(args.event, template_id, obj)
+            for ref in fo.references:
-            for ref in refs:
+                r = pymisp.add_object_reference(ref)
-                r = pymisp.add_object_reference(obj['uuid'], ref)

examples/generate_file_objects.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from pymisp import MISPEncode
+from pymisp.tools import make_binary_objects
+import argparse
+import json
+
+
+def check():
+    missing_dependencies = {'pydeep': False, 'lief': False, 'magic': False, 'pymisp': False}
+    try:
+        import pymisp  # noqa
+    except ImportError:
+        missing_dependencies['pymisp'] = 'Please install pydeep: pip install pymisp'
+    try:
+        import pydeep  # noqa
+    except ImportError:
+        missing_dependencies['pydeep'] = 'Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git'
+    try:
+        import lief  # noqa
+    except ImportError:
+        missing_dependencies['lief'] = 'Please install lief, documentation here: https://github.com/lief-project/LIEF'
+    try:
+        import magic  # noqa
+    except ImportError:
+        missing_dependencies['magic'] = 'Please install python-magic: pip install python-magic.'
+    return json.dumps(missing_dependencies)
+
+
+def make_objects(path):
+    to_return = {'objects': [], 'references': []}
+    fo, peo, seos = make_binary_objects(path)
+
+    if seos:
+        for s in seos:
+            to_return['objects'].append(s)
+            if s.references:
+                to_return['references'] += s.references
+
+    if peo:
+        to_return['objects'].append(peo)
+        if peo.references:
+            to_return['references'] += peo.references
+
+    if fo:
+        to_return['objects'].append(fo)
+        if fo.references:
+            to_return['references'] += fo.references
+    return json.dumps(to_return, cls=MISPEncode)
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Extract indicators out of binaries and returns MISP objects.')
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument("-p", "--path", help="Path to process.")
+    group.add_argument("-c", "--check", action='store_true', help="Check the dependencies.")
+    args = parser.parse_args()
+
+    if args.check:
+        print(check())
+    if args.path:
+        obj = make_objects(args.path)
+        print(obj)

pymisp/__init__.py

@@ -2,7 +2,7 @@ __version__ = '2.4.77'
 
 from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey
 from .api import PyMISP
-from .abstract import AbstractMISP
+from .abstract import AbstractMISP, MISPEncode
 from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull
 from .tools import Neo4j
 from .tools import stix

pymisp/abstract.py

@@ -4,9 +4,18 @@
 import six  # Remove that import when discarding python2 support.
 import abc
 import json
+from json import JSONEncoder
 import collections
 
 
+class MISPEncode(JSONEncoder):
+
+    def default(self, obj):
+        if isinstance(obj, AbstractMISP):
+            return obj.jsonable()
+        return JSONEncoder.default(self, obj)
+
+
 @six.add_metaclass(abc.ABCMeta)   # Remove that line when discarding python2 support.
 # Python3 way: class MISPObjectGenerator(metaclass=abc.ABCMeta):
 class AbstractMISP(collections.MutableMapping):
@@ -48,8 +57,11 @@ class AbstractMISP(collections.MutableMapping):
             to_return[attribute] = val
         return to_return
 
+    def jsonable(self):
+        return self.to_dict()
+
     def to_json(self):
-        return json.dumps(self.to_dict(), indent=4, sort_keys=True)
+        return json.dumps(self.to_dict(), cls=MISPEncode)
 
     def __getitem__(self, key):
         if self.__check_dict_key(key):

pymisp/api.py

@@ -1585,13 +1585,13 @@ class PyMISP(object):
     def add_object(self, event_id, template_id, misp_object):
         session = self.__prepare_session()
         url = urljoin(self.root_url, 'objects/add/{}/{}'.format(event_id, template_id))
-        response = session.post(url, data=json.dumps(misp_object))
+        response = session.post(url, data=misp_object.to_json())
         return self._check_response(response)
 
-    def add_object_reference(self, parent_uuid, misp_object_reference):
+    def add_object_reference(self, misp_object_reference):
         session = self.__prepare_session()
-        url = urljoin(self.root_url, 'object_references/add/{}'.format(parent_uuid))
+        url = urljoin(self.root_url, 'object_references/add')
-        response = session.post(url, data=json.dumps(misp_object_reference))
+        response = session.post(url, data=misp_object_reference.to_json())
         return self._check_response(response)
 
     def get_object_templates_list(self):

pymisp/tools/create_misp_object.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 from pymisp.tools import FileObject, PEObject, MISPObjectException
@@ -17,11 +17,11 @@ class FileTypeNotImplemented(MISPObjectException):
 def make_pe_objects(lief_parsed, misp_file):
     misp_pe = PEObject(parsed=lief_parsed)
     misp_file.add_reference(misp_pe.uuid, 'included-in', 'PE indicators')
-    file_object = misp_file.dump()
+    file_object = misp_file
-    pe_object = misp_pe.dump()
+    pe_object = misp_pe
     pe_sections = []
     for s in misp_pe.sections:
-        pe_sections.append(s.dump())
+        pe_sections.append(s)
     return file_object, pe_object, pe_sections
 
 
@@ -45,5 +45,5 @@ def make_binary_objects(filepath):
         print('\tParser error: ', e)
     except FileTypeNotImplemented as e:
         print(e)
-    file_object = misp_file.dump()
+    file_object = misp_file.to_json()
     return file_object, None, None

pymisp/tools/fileobject.py

@@ -7,6 +7,7 @@ from io import BytesIO
 from hashlib import md5, sha1, sha256, sha512
 import math
 from collections import Counter
+import warnings
 
 try:
     import pydeep
@@ -25,9 +26,9 @@ class FileObject(MISPObjectGenerator):
 
     def __init__(self, filepath=None, pseudofile=None, filename=None):
         if not HAS_PYDEEP:
-            raise ImportError("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git")
+            warnings.warn("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git")
         if not HAS_MAGIC:
-            raise ImportError("Please install python-magic: pip install python-magic.")
+            warnings.warn("Please install python-magic: pip install python-magic.")
         if filepath:
             self.filepath = filepath
             self.filename = os.path.basename(self.filepath)
@@ -46,16 +47,18 @@ class FileObject(MISPObjectGenerator):
 
     def generate_attributes(self):
         self._create_attribute('filename', value=self.filename)
-        self._create_attribute('size-in-bytes', value=len(self.data))
+        size = self._create_attribute('size-in-bytes', value=len(self.data))
-        if getattr(self, 'size-in-bytes').value > 0:
+        if int(size.value) > 0:
             self._create_attribute('entropy', value=self.__entropy_H(self.data))
             self._create_attribute('md5', value=md5(self.data).hexdigest())
             self._create_attribute('sha1', value=sha1(self.data).hexdigest())
             self._create_attribute('sha256', value=sha256(self.data).hexdigest())
             self._create_attribute('sha512', value=sha512(self.data).hexdigest())
+            self._create_attribute('malware-sample', value=self.filename, data=self.pseudofile)
+            if HAS_MAGIC:
                 self._create_attribute('mimetype', value=magic.from_buffer(self.data))
+            if HAS_PYDEEP:
                 self._create_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode())
-            self._create_attribute('malware-sample', value=self.filename.value, data=self.pseudofile)
 
     def __entropy_H(self, data):
         """Calculate the entropy of a chunk of data."""

pymisp/tools/objectgenerator.py

@@ -26,16 +26,53 @@ if six.PY2:
 
 class MISPObjectReference(AbstractMISP):
 
-    attributes = ['uuid', 'relationship_type', 'comment']
+    attributes = ['source_uuid', 'destination_uuid', 'relationship_type', 'comment']
 
-    def __init__(self, uuid, relationship_type, comment=None):
+    def __init__(self, source_uuid, destination_uuid, relationship_type, comment=None):
-        self['uuid'] = uuid
+        self.source_uuid = source_uuid
-        self['relationship_type'] = relationship_type
+        self.destination_uuid = destination_uuid
-        self['comment'] = comment
+        self.relationship_type = relationship_type
+        self.comment = comment
+
+
+class MISPObjectAttribute(AbstractMISP):
+
+    # This list is very limited and hardcoded to fit the current needs (file/pe/pesection creation): MISPAttriute will follow the
+    # same spec and just add one attribute: object_relation
+    attributes = ['object_relation', 'value', 'type', 'category', 'disable_correlation', 'to_ids',
+                  'data', 'encrypt', 'distribution', 'comment']
+
+    def __init__(self, definition, object_relation, value, **kwargs):
+        self.object_relation = object_relation
+        self.value = value
+        # Initialize the new MISPAttribute
+        # Get the misp attribute type from the definition
+        self.type = kwargs.pop('type', None)
+        if self.type is None:
+            self.type = definition['misp-attribute']
+        self.disable_correlation = kwargs.pop('disable_correlation', None)
+        if self.disable_correlation is None:
+            # The correlation can be disabled by default in the object definition.
+            # Use this value if it isn't overloaded by the object
+            self.disable_correlation = definition.get('disable_correlation')
+        self.to_ids = kwargs.pop('to_ids', None)
+        if self.to_ids is None:
+            # Same for the to_ids flag
+            self.to_ids = definition.get('to_ids')
+        # Initialise rest of the values
+        for k, v in kwargs.items():
+            self[k] = v
+        # FIXME: dirty hack until all the classes are ported to the new format but we get the default values
+        temp_attribute = MISPAttribute()
+        temp_attribute.set_all_values(**self)
+        # Update default values
+        self.from_dict(**temp_attribute.to_dict())
 
 
 class MISPObjectGenerator(AbstractMISP):
 
+    attributes = ['name', 'meta-category', 'uuid', 'description', 'version', 'Attribute']
+
     def __init__(self, template_dir):
         """This class is used to fill a new MISP object with the default values defined in the object template
             * template is the path to the template within the misp-object repository
@@ -46,53 +83,38 @@ class MISPObjectGenerator(AbstractMISP):
             'data', 'misp-objects', 'objects')
         with open(os.path.join(self.misp_objects_path, template_dir, 'definition.json'), 'r') as f:
             self.definition = json.load(f)
-        self.attributes = self.definition['attributes'].keys()
+        self.object_attributes = self.definition['attributes'].keys()
         self.misp_event = MISPEvent()
+        self.name = self.definition['name']
+        setattr(self, 'meta-category', self.definition['meta-category'])
         self.uuid = str(uuid.uuid4())
+        self.description = self.definition['description']
+        self.version = self.definition['version']
+        self.Attribute = []
         self.references = []
 
     def _create_attribute(self, object_type, **value):
         if value.get('value') is None:
             return None
-        # Initialize the new MISPAttribute
+        attribute = MISPObjectAttribute(self.definition['attributes'][object_type], object_type, **value)
-        # Get the misp attribute type from the definition
+        self.Attribute.append(attribute)
-        value['type'] = self.definition['attributes'][object_type]['misp-attribute']
+        return attribute
-        if value.get('disable_correlation') is None:
-            # The correlation can be disabled by default in the object definition.
-            # Use this value if it isn't overloaded by the object
-            value['disable_correlation'] = self.definition['attributes'][object_type].get('disable_correlation')
-        if value.get('to_ids') is None:
-            # Same for the to_ids flag
-            value['to_ids'] = self.definition['attributes'][object_type].get('to_ids')
-        # Set all the values in the MISP attribute
-        attribute = MISPAttribute(self.misp_event.describe_types)
-        attribute.set_all_values(**value)
-        self[object_type] = attribute
 
-    def dump(self, strict=True):
+    def to_dict(self, strict=True):
-        """Create a new object with the values gathered by the sub-class, use the default values from the template if needed"""
         if strict:
             self._validate()
-        # Create an empty object based om the object definition
+        return super(MISPObjectGenerator, self).to_dict()
-        new_object = self.__new_empty_object(self.definition)
+
-        for object_type, attribute in self.items():
+    def to_json(self, strict=True):
-            # Add all the values as MISPAttributes to the current object
+        if strict:
-            if attribute.value is None:
+            self._validate()
-                continue
+        return super(MISPObjectGenerator, self).to_json()
-            # Finalize the actual MISP Object
-            # FIXME: This only works on python >= 3.5
-            # new_object['Attribute'].append({'object_relation': object_type, **attribute._json()})
-            # ### BEGIN ####
-            # Because we still need to support old python.
-            temp_attribute = {'object_relation': object_type}
-            temp_attribute.update(attribute._json())
-            new_object['Attribute'].append(temp_attribute)
-            # ###  END  ####
-        return new_object, [r.to_dict() for r in self.references]
 
     def _validate(self):
         """Make sure the object we're creating has the required fields"""
-        all_attribute_names = set(self.keys())
+        all_attribute_names = set()
+        for a in self.Attribute:
+            all_attribute_names.add(a.object_relation)
         if self.definition.get('requiredOneOf'):
             if not set(self.definition['requiredOneOf']) & all_attribute_names:
                 raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self.definition['requiredOneOf'])))
@@ -102,15 +124,9 @@ class MISPObjectGenerator(AbstractMISP):
                     raise InvalidMISPObject('{} is required is required'.format(r))
         return True
 
-    def add_reference(self, uuid, relationship_type, comment=None):
+    def add_reference(self, destination_uuid, relationship_type, comment=None):
         """Add a link (uuid) to an other object"""
-        self.references.append(MISPObjectReference(uuid, relationship_type, comment))
+        self.references.append(MISPObjectReference(self.uuid, destination_uuid, relationship_type, comment))
-
-    def __new_empty_object(self, object_definiton):
-        """Create a new empty object out of the template"""
-        return {'name': object_definiton['name'], 'meta-category': object_definiton['meta-category'],
-                'uuid': self.uuid, 'description': object_definiton['description'],
-                'version': object_definiton['version'], 'Attribute': []}
 
     @abc.abstractmethod
     def generate_attributes(self):

pymisp/tools/peobject.py

@@ -5,6 +5,7 @@ from pymisp.tools import MISPObjectGenerator
 from io import BytesIO
 from hashlib import md5, sha1, sha256, sha512
 from datetime import datetime
+import warnings
 
 
 try:
@@ -24,7 +25,7 @@ class PEObject(MISPObjectGenerator):
 
     def __init__(self, parsed=None, filepath=None, pseudofile=None):
         if not HAS_PYDEEP:
-            raise ImportError("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git")
+            warnings.warn("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git")
         if not HAS_LIEF:
             raise ImportError('Please install lief, documentation here: https://github.com/lief-project/LIEF')
         if pseudofile:
@@ -120,8 +121,8 @@ class PESectionObject(MISPObjectGenerator):
 
     def generate_attributes(self):
         self._create_attribute('name', value=self.section.name)
-        self._create_attribute('size-in-bytes', value=self.section.size)
+        size = self._create_attribute('size-in-bytes', value=self.section.size)
-        if getattr(self, 'size-in-bytes').value > 0:
+        if int(size.value) > 0:
             self._create_attribute('entropy', value=self.section.entropy)
             self._create_attribute('md5', value=md5(self.data).hexdigest())
             self._create_attribute('sha1', value=sha1(self.data).hexdigest())