Initial commit supporting MISP Objects

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "pymisp/data/misp-objects"]
+	path = pymisp/data/misp-objects
+	url = https://github.com/MISP/misp-objects

pymisp/__init__.py

@@ -3,5 +3,6 @@ __version__ = '2.4.77'
 from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey
 from .api import PyMISP
 from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull
-from .tools.neo4j import Neo4j
+from .tools import Neo4j
 from .tools import stix
+from .tools import MISPObjectGenerator

pymisp/api.py

@@ -1578,6 +1578,18 @@ class PyMISP(object):
         response = session.post(url)
         return self._check_response(response)
 
+    # ###################
+    # ###   Objects   ###
+    # ###################
+
+    def add_object(self, event_id, template_id, misp_object):
+        session = self.__prepare_session()
+        url = urljoin(self.root_url, 'objectTemplates/add/{}/{}'.format(event_id, template_id))
+        if not misp_object.get('object'):
+            misp_object = {'object': misp_object}
+        response = session.post(url, data=json.dumps(misp_object))
+        return self._check_response(response)
+
     # ###########################
     # ####### Deprecated ########
     # ###########################

pymisp/data/misp-objects

@@ -0,0 +1 @@
+Subproject commit ca24684e2f49bcfbd886212ff003472716c26de9

pymisp/tools/__init__.py

@@ -0,0 +1,5 @@
+from .neo4j import Neo4j
+from .objectgenerator import MISPObjectGenerator, MISPObjectException, InvalidMISPObject
+from .fileobject import FileObject
+from .peobject import PEObject, PESectionObject
+from .create_misp_object import make_binary_objects

pymisp/tools/create_misp_object.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from pymisp.tools import FileObject, PEObject, MISPObjectException
+
+try:
+    import lief
+    HAS_LIEF = True
+except ImportError:
+    HAS_LIEF = False
+
+
+class FileTypeNotImplemented(MISPObjectException):
+    pass
+
+
+def make_pe_objects(lief_parsed, misp_file):
+    misp_pe = PEObject(parsed=lief_parsed)
+    misp_file.add_link(misp_pe.uuid, 'PE indicators')
+    file_object = misp_file.dump()
+    pe_object = misp_pe.dump()
+    pe_sections = []
+    for s in misp_pe.sections:
+        pe_sections.append(s.dump())
+    return file_object, pe_object, pe_sections
+
+
+def make_binary_objects(filepath):
+    if not HAS_LIEF:
+        raise ImportError('Please install lief, documentation here: https://github.com/lief-project/LIEF')
+    misp_file = FileObject(filepath)
+    try:
+        lief_parsed = lief.parse(filepath)
+        if isinstance(lief_parsed, lief.PE.Binary):
+            make_pe_objects(lief_parsed, misp_file)
+        elif isinstance(lief_parsed, lief.ELF.Binary):
+            raise FileTypeNotImplemented('ELF not implemented yet.')
+        elif isinstance(lief_parsed, lief.MachO.Binary):
+            raise FileTypeNotImplemented('MachO not implemented yet.')
+    except lief.bad_format as e:
+        print('\tBad format: ', e)
+    except lief.bad_file as e:
+        print('\tBad file: ', e)
+    except lief.parser_error as e:
+        print('\tParser error: ', e)
+    except FileTypeNotImplemented as e:
+        print(e)
+    file_object = misp_file.dump()
+    return file_object, None, None

pymisp/tools/fileobject.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from pymisp.tools import MISPObjectGenerator
+import os
+from io import BytesIO
+from hashlib import md5, sha1, sha256, sha512
+import math
+from collections import Counter
+
+try:
+    import pydeep
+    HAS_PYDEEP = True
+except ImportError:
+    HAS_PYDEEP = False
+
+try:
+    import magic
+    HAS_MAGIC = True
+except ImportError:
+    HAS_MAGIC = False
+
+
+class FileObject(MISPObjectGenerator):
+
+    def __init__(self, filepath=None, pseudofile=None, filename=None):
+        if not HAS_PYDEEP:
+            raise ImportError("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git")
+        if not HAS_MAGIC:
+            raise ImportError("Please install python-magic: pip install python-magic.")
+        if filepath:
+            self.filepath = filepath
+            self.filename = os.path.basename(self.filepath)
+            with open(filepath, 'rb') as f:
+                self.pseudofile = BytesIO(f.read())
+        elif pseudofile and isinstance(pseudofile, BytesIO):
+            # WARNING: lief.parse requires a path
+            self.filepath = None
+            self.pseudofile = pseudofile
+            self.filename = filename
+        else:
+            raise Exception('File buffer (BytesIO) or a path is required.')
+        MISPObjectGenerator.__init__(self, 'file')
+        self.data = self.pseudofile.getvalue()
+        self.generate_attributes()
+
+    def generate_attributes(self):
+        self.size = len(self.data)
+        if self.size > 0:
+            self.entropy = self.__entropy_H(self.data)
+            self.md5 = md5(self.data).hexdigest()
+            self.sha1 = sha1(self.data).hexdigest()
+            self.sha256 = sha256(self.data).hexdigest()
+            self.sha512 = sha512(self.data).hexdigest()
+            self.filetype = magic.from_buffer(self.data)
+            self.ssdeep = pydeep.hash_buf(self.data).decode()
+
+    def __entropy_H(self, data):
+        """Calculate the entropy of a chunk of data."""
+        # NOTE: copy of the entropy function from pefile
+
+        if len(data) == 0:
+            return 0.0
+
+        occurences = Counter(bytearray(data))
+
+        entropy = 0
+        for x in occurences.values():
+            p_x = float(x) / len(data)
+            entropy -= p_x * math.log(p_x, 2)
+
+        return entropy
+
+    def dump(self):
+        file_object = {}
+        file_object['filename'] = {'value': self.filename}
+        file_object['size-in-bytes'] = {'value': self.size}
+        if self.size > 0:
+            file_object['entropy'] = {'value': self.entropy}
+            file_object['ssdeep'] = {'value': self.ssdeep}
+            file_object['sha512'] = {'value': self.sha512}
+            file_object['md5'] = {'value': self.md5}
+            file_object['sha1'] = {'value': self.sha1}
+            file_object['sha256'] = {'value': self.sha256}
+            file_object['malware-sample'] = {'value': '{}|{}'.format(self.filename, self.md5), 'data': self.pseudofile}
+            # file_object['authentihash'] = self.
+            # file_object['sha-224'] = self.
+            # file_object['sha-384'] = self.
+            # file_object['sha512/224'] = self.
+            # file_object['sha512/256'] = self.
+            # file_object['tlsh'] = self.
+        return self._fill_object(file_object)

pymisp/tools/objectgenerator.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from pymisp import MISPEvent, MISPAttribute
+import os
+import json
+import uuid
+import abc
+import sys
+
+
+class MISPObjectException(Exception):
+    pass
+
+
+class InvalidMISPObject(MISPObjectException):
+    """Exception raised when an object doesn't contains the required field(s)"""
+    pass
+
+
+class MISPObjectGenerator(metaclass=abc.ABCMeta):
+
+    def __init__(self, template_dir):
+        """This class is used to fill a new MISP object with the default values defined in the object template
+            * template is the path to the template within the misp-object repository
+            * misp_objects_path is the path to the misp-object repository
+        """
+        self.misp_objects_path = os.path.join(
+            os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)),
+            'data', 'misp-objects', 'objects')
+        with open(os.path.join(self.misp_objects_path, template_dir, 'definition.json'), 'r') as f:
+            self.definition = json.load(f)
+        self.misp_event = MISPEvent()
+        self.uuid = str(uuid.uuid4())
+        self.links = []
+
+    def _fill_object(self, values, strict=True):
+        """Create a new object with the values gathered by the sub-class, use the default values from the template if needed"""
+        if strict:
+            self._validate(values)
+        # Create an empty object based om the object definition
+        new_object = self.__new_empty_object(self.definition)
+        if self.links:
+            # Set the links to other objects
+            new_object["ObjectReference"] = []
+            for link in self.links:
+                uuid, comment = link
+                new_object['ObjectReference'].append({'referenced_object_uuid': uuid, 'comment': comment})
+        for object_type, value in values.items():
+            # Add all the values as MISPAttributes to the current object
+            if value.get('value') is None:
+                continue
+            # Initialize the new MISPAttribute
+            attribute = MISPAttribute(self.misp_event.describe_types)
+            # Get the misp attribute type from the definition
+            value['type'] = self.definition['attributes'][object_type]['misp-attribute']
+            if value.get('disable_correlation') is None:
+                # The correlation can be disabled by default in the object definition.
+                # Use this value if it isn't overloaded by the object
+                value['disable_correlation'] = self.definition['attributes'][object_type].get('disable_correlation')
+            if value.get('to_ids') is None:
+                # Same for the to_ids flag
+                value['to_ids'] = self.definition['attributes'][object_type].get('to_ids')
+            # Set all the values in the MISP attribute
+            attribute.set_all_values(**value)
+            # Finalize the actual MISP Object
+            new_object['ObjectAttribute'].append({'type': object_type, 'Attribute': attribute._json()})
+        return new_object
+
+    def _validate(self, dump):
+        """Make sure the object we're creating has the required fields"""
+        all_attribute_names = set(dump.keys())
+        if self.definition.get('requiredOneOf'):
+            if not set(self.definition['requiredOneOf']) & all_attribute_names:
+                raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self.definition['requiredOneOf'])))
+        if self.definition.get('required'):
+            for r in self.definition.get('required'):
+                if r not in all_attribute_names:
+                    raise InvalidMISPObject('{} is required is required'.format(r))
+        return True
+
+    def add_link(self, uuid, comment=None):
+        """Add a link (uuid) to an other object"""
+        self.links.append((uuid, comment))
+
+    def __new_empty_object(self, object_definiton):
+        """Create a new empty object out of the template"""
+        return {'name': object_definiton['name'], 'meta-category': object_definiton['meta-category'],
+                'uuid': self.uuid, 'description': object_definiton['description'],
+                'version': object_definiton['version'], 'ObjectAttribute': []}
+
+    @abc.abstractmethod
+    def generate_attributes(self):
+        """Contains the logic where all the values of the object are gathered"""
+        pass
+
+    @abc.abstractmethod
+    def dump(self):
+        """This method normalize the attributes to add to the object.
+        It returns an python dictionary where the key is the type defined in the object,
+        and the value the value of the MISP Attribute"""
+        pass

pymisp/tools/peobject.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from pymisp.tools import MISPObjectGenerator
+from io import BytesIO
+from hashlib import md5, sha1, sha256, sha512
+from datetime import datetime
+
+
+try:
+    import lief
+    HAS_LIEF = True
+except ImportError:
+    HAS_LIEF = False
+
+try:
+    import pydeep
+    HAS_PYDEEP = True
+except ImportError:
+    HAS_PYDEEP = False
+
+
+class PEObject(MISPObjectGenerator):
+
+    def __init__(self, parsed=None, filepath=None, pseudofile=None):
+        if not HAS_PYDEEP:
+            raise ImportError("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git")
+        if not HAS_LIEF:
+            raise ImportError('Please install lief, documentation here: https://github.com/lief-project/LIEF')
+        if pseudofile:
+            if isinstance(pseudofile, BytesIO):
+                self.pe = lief.PE.parse(raw=pseudofile.getvalue())
+            elif isinstance(pseudofile, bytes):
+                self.pe = lief.PE.parse(raw=pseudofile)
+            else:
+                raise Exception('Pseudo file can be BytesIO or bytes got {}'.format(type(pseudofile)))
+        elif filepath:
+            self.pe = lief.PE.parse(filepath)
+        elif parsed:
+            # Got an already parsed blob
+            if isinstance(parsed, lief.PE.Binary):
+                self.pe = parsed
+            else:
+                raise Exception('Not a lief.PE.Binary: {}'.format(type(parsed)))
+        MISPObjectGenerator.__init__(self, 'pe')
+        self.generate_attributes()
+
+    def _is_exe(self):
+        if not self._is_dll() and not self._is_driver():
+            return self.pe.header.has_characteristic(lief.PE.HEADER_CHARACTERISTICS.EXECUTABLE_IMAGE)
+        return False
+
+    def _is_dll(self):
+        return self.pe.header.has_characteristic(lief.PE.HEADER_CHARACTERISTICS.DLL)
+
+    def _is_driver(self):
+        # List from pefile
+        system_DLLs = set(('ntoskrnl.exe', 'hal.dll', 'ndis.sys', 'bootvid.dll', 'kdcom.dll'))
+        if system_DLLs.intersection([imp.lower() for imp in self.pe.libraries]):
+            return True
+        return False
+
+    def generate_attributes(self):
+        if self._is_dll():
+            self.pe_type = 'dll'
+        elif self._is_driver():
+            self.pe_type = 'driver'
+        elif self._is_exe():
+            self.pe_type = 'exe'
+        else:
+            self.pe_type = 'unknown'
+        # General information
+        self.entrypoint_address = self.pe.entrypoint
+        self.compilation_timestamp = datetime.utcfromtimestamp(self.pe.header.time_date_stamps).isoformat()
+        # self.imphash = self.pe.get_imphash()
+        try:
+            if (self.pe.has_resources and
+                    self.pe.resources_manager.has_version and
+                    self.pe.resources_manager.version.has_string_file_info and
+                    self.pe.resources_manager.version.string_file_info.langcode_items):
+                fileinfo = dict(self.pe.resources_manager.version.string_file_info.langcode_items[0].items.items())
+                self.original_filename = fileinfo.get('OriginalFilename')
+                self.internal_filename = fileinfo.get('InternalName')
+                self.file_description = fileinfo.get('FileDescription')
+                self.file_version = fileinfo.get('FileVersion')
+                self.lang_id = self.pe.resources_manager.version.string_file_info.langcode_items[0].key
+                self.product_name = fileinfo.get('ProductName')
+                self.product_version = fileinfo.get('ProductVersion')
+                self.company_name = fileinfo.get('CompanyName')
+                self.legal_copyright = fileinfo.get('LegalCopyright')
+        except lief.read_out_of_bound:
+            # The file is corrupted
+            pass
+        # Sections
+        self.sections = []
+        if self.pe.sections:
+            pos = 0
+            for section in self.pe.sections:
+                s = PESectionObject(section)
+                self.add_link(s.uuid, 'Section {} of PE'.format(pos))
+                if ((self.entrypoint_address >= section.virtual_address) and
+                        (self.entrypoint_address < (section.virtual_address + section.virtual_size))):
+                    self.entrypoint_section = (section.name, pos)  # Tuple: (section_name, position)
+                pos += 1
+                self.sections.append(s)
+        self.nb_sections = len(self.sections)
+        # TODO: TLSSection / DIRECTORY_ENTRY_TLS
+
+    def dump(self):
+        pe_object = {}
+        pe_object['type'] = {'value': self.pe_type}
+        if hasattr(self, 'imphash'):
+            pe_object['imphash'] = {'value': self.imphash}
+        if hasattr(self, 'original_filename'):
+            pe_object['original-filename'] = {'value': self.original_filename}
+        if hasattr(self, 'internal_filename'):
+            pe_object['internal-filename'] = {'value': self.internal_filename}
+        if hasattr(self, 'compilation_timestamp'):
+            pe_object['compilation-timestamp'] = {'value': self.compilation_timestamp}
+        if hasattr(self, 'entrypoint_section'):
+            pe_object['entrypoint-section|position'] = {'value': '{}|{}'.format(*self.entrypoint_section)}
+        if hasattr(self, 'entrypoint_address'):
+            pe_object['entrypoint-address'] = {'value': self.entrypoint_address}
+        if hasattr(self, 'file_description'):
+            pe_object['file-description'] = {'value': self.file_description}
+        if hasattr(self, 'file_version'):
+            pe_object['file-version'] = {'value': self.file_version}
+        if hasattr(self, 'lang_id'):
+            pe_object['lang-id'] = {'value': self.lang_id}
+        if hasattr(self, 'product_name'):
+            pe_object['product-name'] = {'value': self.product_name}
+        if hasattr(self, 'product_version'):
+            pe_object['product-version'] = {'value': self.product_version}
+        if hasattr(self, 'company_name'):
+            pe_object['company-name'] = {'value': self.company_name}
+        if hasattr(self, 'nb_sections'):
+            pe_object['number-sections'] = {'value': self.nb_sections}
+        return self._fill_object(pe_object)
+
+
+class PESectionObject(MISPObjectGenerator):
+
+    def __init__(self, section):
+        MISPObjectGenerator.__init__(self, 'pe-section')
+        self.section = section
+        self.data = bytes(self.section.content)
+        self.generate_attributes()
+
+    def generate_attributes(self):
+        self.name = self.section.name
+        self.size = self.section.size
+        if self.size > 0:
+            self.entropy = self.section.entropy
+            self.md5 = md5(self.data).hexdigest()
+            self.sha1 = sha1(self.data).hexdigest()
+            self.sha256 = sha256(self.data).hexdigest()
+            self.sha512 = sha512(self.data).hexdigest()
+            if HAS_PYDEEP:
+                self.ssdeep = pydeep.hash_buf(self.data).decode()
+
+    def dump(self):
+        section = {}
+        section['name'] = {'value': self.name}
+        section['size-in-bytes'] = {'value': self.size}
+        if self.size > 0:
+            section['entropy'] = {'value': self.entropy}
+            section['md5'] = {'value': self.md5}
+            section['sha1'] = {'value': self.sha1}
+            section['sha256'] = {'value': self.sha256}
+            section['sha512'] = {'value': self.sha512}
+            section['ssdeep'] = {'value': self.ssdeep}
+        return self._fill_object(section)

pymisp/tools/prepare_misp_object.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from pymisp import PyMISP
+from pymisp.tools import FileObject, PEObject
+from pymisp.tools import make_binary_objects
+import traceback
+
+
+try:
+    import lief
+    HAS_LIEF = True
+except ImportError:
+    HAS_LIEF = False
+    raise ImportError("Please install lief: https://github.com/lief-project/LIEF")
+
+
+if __name__ == '__main__':
+    pymisp = PyMISP('https://mispbeta.circl.lu', 'et9ZEgn70YJ6URkCr6741LpJNAVUMYD1rM063od3')
+
+
+    # fo, peo, seos = make_objects('/home/raphael/.viper/projects/troopers17/vt_samples/1189/566ab945f61be016bfd9e83cc1b64f783b9b8deb891e6d504d3442bc8281b092')
+    import glob
+    for f in glob.glob('/home/raphael/.viper/projects/troopers17/vt_samples/*/*'):
+    #for f in glob.glob('/home/raphael/gits/pefile-tests/tests/corkami/*/*.exe'):
+    #for f in glob.glob('/home/raphael/gits/pefile-tests/tests/corkami/pocs/version_mini.exe'):
+    #for f in glob.glob('/home/raphael/gits/pefile-tests/tests/corkami/pocs/version_cust.exe'):
+    #for f in glob.glob('/home/raphael/gits/pefile-tests/tests/data/*.dll'):
+        print('\n', f)
+        try:
+            fo, peo, seos = make_binary_objects(f)
+        except Exception as e:
+            traceback.print_exc()
+            continue
+        continue
+        if fo:
+            response = pymisp.add_object(2221, 7, fo)
+            print(response)
+        if peo:
+            pymisp.add_object(2221, 11, peo)
+        if seos:
+            for s in seos:
+                pymisp.add_object(2221, 12, s)
+
+        #with open('fileobj.json', 'w') as f:
+        #    json.dump(fo, f)
+        #with open('peobj.json', 'w') as f:
+        #    json.dump(peo, f)
+        #with open('seobj.json', 'w') as f:
+        #    json.dump(seos, f)
+        break

setup.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 from setuptools import setup
 import pymisp
@@ -29,5 +29,6 @@ setup(
     test_suite="tests",
     install_requires=['requests', 'python-dateutil', 'jsonschema'],
     include_package_data=True,
-    package_data={'data': ['schema.json', 'schema-lax.json', 'describeTypes.json']},
+    package_data={'pymisp': ['data/*.json', 'data/misp-objects/schema.json',
+                             'data/misp-objects/objects/*/definition.json']},
 )