diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..d0e9062 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "pymisp/data/misp-objects"] + path = pymisp/data/misp-objects + url = https://github.com/MISP/misp-objects diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 9473142..8bf84d1 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -3,5 +3,6 @@ __version__ = '2.4.77' from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey from .api import PyMISP from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull -from .tools.neo4j import Neo4j +from .tools import Neo4j from .tools import stix +from .tools import MISPObjectGenerator diff --git a/pymisp/api.py b/pymisp/api.py index 3c17d18..ad9d685 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1578,6 +1578,18 @@ class PyMISP(object): response = session.post(url) return self._check_response(response) + # ################### + # ### Objects ### + # ################### + + def add_object(self, event_id, template_id, misp_object): + session = self.__prepare_session() + url = urljoin(self.root_url, 'objectTemplates/add/{}/{}'.format(event_id, template_id)) + if not misp_object.get('object'): + misp_object = {'object': misp_object} + response = session.post(url, data=json.dumps(misp_object)) + return self._check_response(response) + # ########################### # ####### Deprecated ######## # ########################### diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects new file mode 160000 index 0000000..ca24684 --- /dev/null +++ b/pymisp/data/misp-objects @@ -0,0 +1 @@ +Subproject commit ca24684e2f49bcfbd886212ff003472716c26de9 diff --git a/pymisp/tools/__init__.py b/pymisp/tools/__init__.py index e69de29..dc1748c 100644 --- a/pymisp/tools/__init__.py +++ b/pymisp/tools/__init__.py @@ -0,0 +1,5 @@ +from .neo4j import Neo4j +from .objectgenerator import MISPObjectGenerator, MISPObjectException, InvalidMISPObject +from .fileobject import FileObject +from .peobject import PEObject, PESectionObject +from .create_misp_object import make_binary_objects diff --git a/pymisp/tools/create_misp_object.py b/pymisp/tools/create_misp_object.py new file mode 100644 index 0000000..cb2f917 --- /dev/null +++ b/pymisp/tools/create_misp_object.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from pymisp.tools import FileObject, PEObject, MISPObjectException + +try: + import lief + HAS_LIEF = True +except ImportError: + HAS_LIEF = False + + +class FileTypeNotImplemented(MISPObjectException): + pass + + +def make_pe_objects(lief_parsed, misp_file): + misp_pe = PEObject(parsed=lief_parsed) + misp_file.add_link(misp_pe.uuid, 'PE indicators') + file_object = misp_file.dump() + pe_object = misp_pe.dump() + pe_sections = [] + for s in misp_pe.sections: + pe_sections.append(s.dump()) + return file_object, pe_object, pe_sections + + +def make_binary_objects(filepath): + if not HAS_LIEF: + raise ImportError('Please install lief, documentation here: https://github.com/lief-project/LIEF') + misp_file = FileObject(filepath) + try: + lief_parsed = lief.parse(filepath) + if isinstance(lief_parsed, lief.PE.Binary): + make_pe_objects(lief_parsed, misp_file) + elif isinstance(lief_parsed, lief.ELF.Binary): + raise FileTypeNotImplemented('ELF not implemented yet.') + elif isinstance(lief_parsed, lief.MachO.Binary): + raise FileTypeNotImplemented('MachO not implemented yet.') + except lief.bad_format as e: + print('\tBad format: ', e) + except lief.bad_file as e: + print('\tBad file: ', e) + except lief.parser_error as e: + print('\tParser error: ', e) + except FileTypeNotImplemented as e: + print(e) + file_object = misp_file.dump() + return file_object, None, None diff --git a/pymisp/tools/fileobject.py b/pymisp/tools/fileobject.py new file mode 100644 index 0000000..6035b43 --- /dev/null +++ b/pymisp/tools/fileobject.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from pymisp.tools import MISPObjectGenerator +import os +from io import BytesIO +from hashlib import md5, sha1, sha256, sha512 +import math +from collections import Counter + +try: + import pydeep + HAS_PYDEEP = True +except ImportError: + HAS_PYDEEP = False + +try: + import magic + HAS_MAGIC = True +except ImportError: + HAS_MAGIC = False + + +class FileObject(MISPObjectGenerator): + + def __init__(self, filepath=None, pseudofile=None, filename=None): + if not HAS_PYDEEP: + raise ImportError("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") + if not HAS_MAGIC: + raise ImportError("Please install python-magic: pip install python-magic.") + if filepath: + self.filepath = filepath + self.filename = os.path.basename(self.filepath) + with open(filepath, 'rb') as f: + self.pseudofile = BytesIO(f.read()) + elif pseudofile and isinstance(pseudofile, BytesIO): + # WARNING: lief.parse requires a path + self.filepath = None + self.pseudofile = pseudofile + self.filename = filename + else: + raise Exception('File buffer (BytesIO) or a path is required.') + MISPObjectGenerator.__init__(self, 'file') + self.data = self.pseudofile.getvalue() + self.generate_attributes() + + def generate_attributes(self): + self.size = len(self.data) + if self.size > 0: + self.entropy = self.__entropy_H(self.data) + self.md5 = md5(self.data).hexdigest() + self.sha1 = sha1(self.data).hexdigest() + self.sha256 = sha256(self.data).hexdigest() + self.sha512 = sha512(self.data).hexdigest() + self.filetype = magic.from_buffer(self.data) + self.ssdeep = pydeep.hash_buf(self.data).decode() + + def __entropy_H(self, data): + """Calculate the entropy of a chunk of data.""" + # NOTE: copy of the entropy function from pefile + + if len(data) == 0: + return 0.0 + + occurences = Counter(bytearray(data)) + + entropy = 0 + for x in occurences.values(): + p_x = float(x) / len(data) + entropy -= p_x * math.log(p_x, 2) + + return entropy + + def dump(self): + file_object = {} + file_object['filename'] = {'value': self.filename} + file_object['size-in-bytes'] = {'value': self.size} + if self.size > 0: + file_object['entropy'] = {'value': self.entropy} + file_object['ssdeep'] = {'value': self.ssdeep} + file_object['sha512'] = {'value': self.sha512} + file_object['md5'] = {'value': self.md5} + file_object['sha1'] = {'value': self.sha1} + file_object['sha256'] = {'value': self.sha256} + file_object['malware-sample'] = {'value': '{}|{}'.format(self.filename, self.md5), 'data': self.pseudofile} + # file_object['authentihash'] = self. + # file_object['sha-224'] = self. + # file_object['sha-384'] = self. + # file_object['sha512/224'] = self. + # file_object['sha512/256'] = self. + # file_object['tlsh'] = self. + return self._fill_object(file_object) diff --git a/pymisp/tools/objectgenerator.py b/pymisp/tools/objectgenerator.py new file mode 100644 index 0000000..c0a7c63 --- /dev/null +++ b/pymisp/tools/objectgenerator.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pymisp import MISPEvent, MISPAttribute +import os +import json +import uuid +import abc +import sys + + +class MISPObjectException(Exception): + pass + + +class InvalidMISPObject(MISPObjectException): + """Exception raised when an object doesn't contains the required field(s)""" + pass + + +class MISPObjectGenerator(metaclass=abc.ABCMeta): + + def __init__(self, template_dir): + """This class is used to fill a new MISP object with the default values defined in the object template + * template is the path to the template within the misp-object repository + * misp_objects_path is the path to the misp-object repository + """ + self.misp_objects_path = os.path.join( + os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)), + 'data', 'misp-objects', 'objects') + with open(os.path.join(self.misp_objects_path, template_dir, 'definition.json'), 'r') as f: + self.definition = json.load(f) + self.misp_event = MISPEvent() + self.uuid = str(uuid.uuid4()) + self.links = [] + + def _fill_object(self, values, strict=True): + """Create a new object with the values gathered by the sub-class, use the default values from the template if needed""" + if strict: + self._validate(values) + # Create an empty object based om the object definition + new_object = self.__new_empty_object(self.definition) + if self.links: + # Set the links to other objects + new_object["ObjectReference"] = [] + for link in self.links: + uuid, comment = link + new_object['ObjectReference'].append({'referenced_object_uuid': uuid, 'comment': comment}) + for object_type, value in values.items(): + # Add all the values as MISPAttributes to the current object + if value.get('value') is None: + continue + # Initialize the new MISPAttribute + attribute = MISPAttribute(self.misp_event.describe_types) + # Get the misp attribute type from the definition + value['type'] = self.definition['attributes'][object_type]['misp-attribute'] + if value.get('disable_correlation') is None: + # The correlation can be disabled by default in the object definition. + # Use this value if it isn't overloaded by the object + value['disable_correlation'] = self.definition['attributes'][object_type].get('disable_correlation') + if value.get('to_ids') is None: + # Same for the to_ids flag + value['to_ids'] = self.definition['attributes'][object_type].get('to_ids') + # Set all the values in the MISP attribute + attribute.set_all_values(**value) + # Finalize the actual MISP Object + new_object['ObjectAttribute'].append({'type': object_type, 'Attribute': attribute._json()}) + return new_object + + def _validate(self, dump): + """Make sure the object we're creating has the required fields""" + all_attribute_names = set(dump.keys()) + if self.definition.get('requiredOneOf'): + if not set(self.definition['requiredOneOf']) & all_attribute_names: + raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self.definition['requiredOneOf']))) + if self.definition.get('required'): + for r in self.definition.get('required'): + if r not in all_attribute_names: + raise InvalidMISPObject('{} is required is required'.format(r)) + return True + + def add_link(self, uuid, comment=None): + """Add a link (uuid) to an other object""" + self.links.append((uuid, comment)) + + def __new_empty_object(self, object_definiton): + """Create a new empty object out of the template""" + return {'name': object_definiton['name'], 'meta-category': object_definiton['meta-category'], + 'uuid': self.uuid, 'description': object_definiton['description'], + 'version': object_definiton['version'], 'ObjectAttribute': []} + + @abc.abstractmethod + def generate_attributes(self): + """Contains the logic where all the values of the object are gathered""" + pass + + @abc.abstractmethod + def dump(self): + """This method normalize the attributes to add to the object. + It returns an python dictionary where the key is the type defined in the object, + and the value the value of the MISP Attribute""" + pass diff --git a/pymisp/tools/peobject.py b/pymisp/tools/peobject.py new file mode 100644 index 0000000..a72062b --- /dev/null +++ b/pymisp/tools/peobject.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from pymisp.tools import MISPObjectGenerator +from io import BytesIO +from hashlib import md5, sha1, sha256, sha512 +from datetime import datetime + + +try: + import lief + HAS_LIEF = True +except ImportError: + HAS_LIEF = False + +try: + import pydeep + HAS_PYDEEP = True +except ImportError: + HAS_PYDEEP = False + + +class PEObject(MISPObjectGenerator): + + def __init__(self, parsed=None, filepath=None, pseudofile=None): + if not HAS_PYDEEP: + raise ImportError("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") + if not HAS_LIEF: + raise ImportError('Please install lief, documentation here: https://github.com/lief-project/LIEF') + if pseudofile: + if isinstance(pseudofile, BytesIO): + self.pe = lief.PE.parse(raw=pseudofile.getvalue()) + elif isinstance(pseudofile, bytes): + self.pe = lief.PE.parse(raw=pseudofile) + else: + raise Exception('Pseudo file can be BytesIO or bytes got {}'.format(type(pseudofile))) + elif filepath: + self.pe = lief.PE.parse(filepath) + elif parsed: + # Got an already parsed blob + if isinstance(parsed, lief.PE.Binary): + self.pe = parsed + else: + raise Exception('Not a lief.PE.Binary: {}'.format(type(parsed))) + MISPObjectGenerator.__init__(self, 'pe') + self.generate_attributes() + + def _is_exe(self): + if not self._is_dll() and not self._is_driver(): + return self.pe.header.has_characteristic(lief.PE.HEADER_CHARACTERISTICS.EXECUTABLE_IMAGE) + return False + + def _is_dll(self): + return self.pe.header.has_characteristic(lief.PE.HEADER_CHARACTERISTICS.DLL) + + def _is_driver(self): + # List from pefile + system_DLLs = set(('ntoskrnl.exe', 'hal.dll', 'ndis.sys', 'bootvid.dll', 'kdcom.dll')) + if system_DLLs.intersection([imp.lower() for imp in self.pe.libraries]): + return True + return False + + def generate_attributes(self): + if self._is_dll(): + self.pe_type = 'dll' + elif self._is_driver(): + self.pe_type = 'driver' + elif self._is_exe(): + self.pe_type = 'exe' + else: + self.pe_type = 'unknown' + # General information + self.entrypoint_address = self.pe.entrypoint + self.compilation_timestamp = datetime.utcfromtimestamp(self.pe.header.time_date_stamps).isoformat() + # self.imphash = self.pe.get_imphash() + try: + if (self.pe.has_resources and + self.pe.resources_manager.has_version and + self.pe.resources_manager.version.has_string_file_info and + self.pe.resources_manager.version.string_file_info.langcode_items): + fileinfo = dict(self.pe.resources_manager.version.string_file_info.langcode_items[0].items.items()) + self.original_filename = fileinfo.get('OriginalFilename') + self.internal_filename = fileinfo.get('InternalName') + self.file_description = fileinfo.get('FileDescription') + self.file_version = fileinfo.get('FileVersion') + self.lang_id = self.pe.resources_manager.version.string_file_info.langcode_items[0].key + self.product_name = fileinfo.get('ProductName') + self.product_version = fileinfo.get('ProductVersion') + self.company_name = fileinfo.get('CompanyName') + self.legal_copyright = fileinfo.get('LegalCopyright') + except lief.read_out_of_bound: + # The file is corrupted + pass + # Sections + self.sections = [] + if self.pe.sections: + pos = 0 + for section in self.pe.sections: + s = PESectionObject(section) + self.add_link(s.uuid, 'Section {} of PE'.format(pos)) + if ((self.entrypoint_address >= section.virtual_address) and + (self.entrypoint_address < (section.virtual_address + section.virtual_size))): + self.entrypoint_section = (section.name, pos) # Tuple: (section_name, position) + pos += 1 + self.sections.append(s) + self.nb_sections = len(self.sections) + # TODO: TLSSection / DIRECTORY_ENTRY_TLS + + def dump(self): + pe_object = {} + pe_object['type'] = {'value': self.pe_type} + if hasattr(self, 'imphash'): + pe_object['imphash'] = {'value': self.imphash} + if hasattr(self, 'original_filename'): + pe_object['original-filename'] = {'value': self.original_filename} + if hasattr(self, 'internal_filename'): + pe_object['internal-filename'] = {'value': self.internal_filename} + if hasattr(self, 'compilation_timestamp'): + pe_object['compilation-timestamp'] = {'value': self.compilation_timestamp} + if hasattr(self, 'entrypoint_section'): + pe_object['entrypoint-section|position'] = {'value': '{}|{}'.format(*self.entrypoint_section)} + if hasattr(self, 'entrypoint_address'): + pe_object['entrypoint-address'] = {'value': self.entrypoint_address} + if hasattr(self, 'file_description'): + pe_object['file-description'] = {'value': self.file_description} + if hasattr(self, 'file_version'): + pe_object['file-version'] = {'value': self.file_version} + if hasattr(self, 'lang_id'): + pe_object['lang-id'] = {'value': self.lang_id} + if hasattr(self, 'product_name'): + pe_object['product-name'] = {'value': self.product_name} + if hasattr(self, 'product_version'): + pe_object['product-version'] = {'value': self.product_version} + if hasattr(self, 'company_name'): + pe_object['company-name'] = {'value': self.company_name} + if hasattr(self, 'nb_sections'): + pe_object['number-sections'] = {'value': self.nb_sections} + return self._fill_object(pe_object) + + +class PESectionObject(MISPObjectGenerator): + + def __init__(self, section): + MISPObjectGenerator.__init__(self, 'pe-section') + self.section = section + self.data = bytes(self.section.content) + self.generate_attributes() + + def generate_attributes(self): + self.name = self.section.name + self.size = self.section.size + if self.size > 0: + self.entropy = self.section.entropy + self.md5 = md5(self.data).hexdigest() + self.sha1 = sha1(self.data).hexdigest() + self.sha256 = sha256(self.data).hexdigest() + self.sha512 = sha512(self.data).hexdigest() + if HAS_PYDEEP: + self.ssdeep = pydeep.hash_buf(self.data).decode() + + def dump(self): + section = {} + section['name'] = {'value': self.name} + section['size-in-bytes'] = {'value': self.size} + if self.size > 0: + section['entropy'] = {'value': self.entropy} + section['md5'] = {'value': self.md5} + section['sha1'] = {'value': self.sha1} + section['sha256'] = {'value': self.sha256} + section['sha512'] = {'value': self.sha512} + section['ssdeep'] = {'value': self.ssdeep} + return self._fill_object(section) diff --git a/pymisp/tools/prepare_misp_object.py b/pymisp/tools/prepare_misp_object.py new file mode 100644 index 0000000..541f6ef --- /dev/null +++ b/pymisp/tools/prepare_misp_object.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from pymisp import PyMISP +from pymisp.tools import FileObject, PEObject +from pymisp.tools import make_binary_objects +import traceback + + +try: + import lief + HAS_LIEF = True +except ImportError: + HAS_LIEF = False + raise ImportError("Please install lief: https://github.com/lief-project/LIEF") + + +if __name__ == '__main__': + pymisp = PyMISP('https://mispbeta.circl.lu', 'et9ZEgn70YJ6URkCr6741LpJNAVUMYD1rM063od3') + + + # fo, peo, seos = make_objects('/home/raphael/.viper/projects/troopers17/vt_samples/1189/566ab945f61be016bfd9e83cc1b64f783b9b8deb891e6d504d3442bc8281b092') + import glob + for f in glob.glob('/home/raphael/.viper/projects/troopers17/vt_samples/*/*'): + #for f in glob.glob('/home/raphael/gits/pefile-tests/tests/corkami/*/*.exe'): + #for f in glob.glob('/home/raphael/gits/pefile-tests/tests/corkami/pocs/version_mini.exe'): + #for f in glob.glob('/home/raphael/gits/pefile-tests/tests/corkami/pocs/version_cust.exe'): + #for f in glob.glob('/home/raphael/gits/pefile-tests/tests/data/*.dll'): + print('\n', f) + try: + fo, peo, seos = make_binary_objects(f) + except Exception as e: + traceback.print_exc() + continue + continue + if fo: + response = pymisp.add_object(2221, 7, fo) + print(response) + if peo: + pymisp.add_object(2221, 11, peo) + if seos: + for s in seos: + pymisp.add_object(2221, 12, s) + + #with open('fileobj.json', 'w') as f: + # json.dump(fo, f) + #with open('peobj.json', 'w') as f: + # json.dump(peo, f) + #with open('seobj.json', 'w') as f: + # json.dump(seos, f) + break diff --git a/setup.py b/setup.py index 7ddac16..ad1f928 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # -*- coding: utf-8 -*- from setuptools import setup import pymisp @@ -29,5 +29,6 @@ setup( test_suite="tests", install_requires=['requests', 'python-dateutil', 'jsonschema'], include_package_data=True, - package_data={'data': ['schema.json', 'schema-lax.json', 'describeTypes.json']}, + package_data={'pymisp': ['data/*.json', 'data/misp-objects/schema.json', + 'data/misp-objects/objects/*/definition.json']}, )