Initial commit supporting MISP Objects

pull/111/head
Raphaël Vinot 2017-07-21 18:47:10 +02:00
parent 14fcc5f586
commit 78488db7aa
11 changed files with 492 additions and 3 deletions

3
.gitmodules vendored Normal file
View File

@ -0,0 +1,3 @@
[submodule "pymisp/data/misp-objects"]
path = pymisp/data/misp-objects
url = https://github.com/MISP/misp-objects

View File

@ -3,5 +3,6 @@ __version__ = '2.4.77'
from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey
from .api import PyMISP
from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull
from .tools.neo4j import Neo4j
from .tools import Neo4j
from .tools import stix
from .tools import MISPObjectGenerator

View File

@ -1578,6 +1578,18 @@ class PyMISP(object):
response = session.post(url)
return self._check_response(response)
# ###################
# ### Objects ###
# ###################
def add_object(self, event_id, template_id, misp_object):
session = self.__prepare_session()
url = urljoin(self.root_url, 'objectTemplates/add/{}/{}'.format(event_id, template_id))
if not misp_object.get('object'):
misp_object = {'object': misp_object}
response = session.post(url, data=json.dumps(misp_object))
return self._check_response(response)
# ###########################
# ####### Deprecated ########
# ###########################

@ -0,0 +1 @@
Subproject commit ca24684e2f49bcfbd886212ff003472716c26de9

View File

@ -0,0 +1,5 @@
from .neo4j import Neo4j
from .objectgenerator import MISPObjectGenerator, MISPObjectException, InvalidMISPObject
from .fileobject import FileObject
from .peobject import PEObject, PESectionObject
from .create_misp_object import make_binary_objects

View File

@ -0,0 +1,49 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from pymisp.tools import FileObject, PEObject, MISPObjectException
try:
import lief
HAS_LIEF = True
except ImportError:
HAS_LIEF = False
class FileTypeNotImplemented(MISPObjectException):
pass
def make_pe_objects(lief_parsed, misp_file):
misp_pe = PEObject(parsed=lief_parsed)
misp_file.add_link(misp_pe.uuid, 'PE indicators')
file_object = misp_file.dump()
pe_object = misp_pe.dump()
pe_sections = []
for s in misp_pe.sections:
pe_sections.append(s.dump())
return file_object, pe_object, pe_sections
def make_binary_objects(filepath):
if not HAS_LIEF:
raise ImportError('Please install lief, documentation here: https://github.com/lief-project/LIEF')
misp_file = FileObject(filepath)
try:
lief_parsed = lief.parse(filepath)
if isinstance(lief_parsed, lief.PE.Binary):
make_pe_objects(lief_parsed, misp_file)
elif isinstance(lief_parsed, lief.ELF.Binary):
raise FileTypeNotImplemented('ELF not implemented yet.')
elif isinstance(lief_parsed, lief.MachO.Binary):
raise FileTypeNotImplemented('MachO not implemented yet.')
except lief.bad_format as e:
print('\tBad format: ', e)
except lief.bad_file as e:
print('\tBad file: ', e)
except lief.parser_error as e:
print('\tParser error: ', e)
except FileTypeNotImplemented as e:
print(e)
file_object = misp_file.dump()
return file_object, None, None

View File

@ -0,0 +1,92 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from pymisp.tools import MISPObjectGenerator
import os
from io import BytesIO
from hashlib import md5, sha1, sha256, sha512
import math
from collections import Counter
try:
import pydeep
HAS_PYDEEP = True
except ImportError:
HAS_PYDEEP = False
try:
import magic
HAS_MAGIC = True
except ImportError:
HAS_MAGIC = False
class FileObject(MISPObjectGenerator):
def __init__(self, filepath=None, pseudofile=None, filename=None):
if not HAS_PYDEEP:
raise ImportError("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git")
if not HAS_MAGIC:
raise ImportError("Please install python-magic: pip install python-magic.")
if filepath:
self.filepath = filepath
self.filename = os.path.basename(self.filepath)
with open(filepath, 'rb') as f:
self.pseudofile = BytesIO(f.read())
elif pseudofile and isinstance(pseudofile, BytesIO):
# WARNING: lief.parse requires a path
self.filepath = None
self.pseudofile = pseudofile
self.filename = filename
else:
raise Exception('File buffer (BytesIO) or a path is required.')
MISPObjectGenerator.__init__(self, 'file')
self.data = self.pseudofile.getvalue()
self.generate_attributes()
def generate_attributes(self):
self.size = len(self.data)
if self.size > 0:
self.entropy = self.__entropy_H(self.data)
self.md5 = md5(self.data).hexdigest()
self.sha1 = sha1(self.data).hexdigest()
self.sha256 = sha256(self.data).hexdigest()
self.sha512 = sha512(self.data).hexdigest()
self.filetype = magic.from_buffer(self.data)
self.ssdeep = pydeep.hash_buf(self.data).decode()
def __entropy_H(self, data):
"""Calculate the entropy of a chunk of data."""
# NOTE: copy of the entropy function from pefile
if len(data) == 0:
return 0.0
occurences = Counter(bytearray(data))
entropy = 0
for x in occurences.values():
p_x = float(x) / len(data)
entropy -= p_x * math.log(p_x, 2)
return entropy
def dump(self):
file_object = {}
file_object['filename'] = {'value': self.filename}
file_object['size-in-bytes'] = {'value': self.size}
if self.size > 0:
file_object['entropy'] = {'value': self.entropy}
file_object['ssdeep'] = {'value': self.ssdeep}
file_object['sha512'] = {'value': self.sha512}
file_object['md5'] = {'value': self.md5}
file_object['sha1'] = {'value': self.sha1}
file_object['sha256'] = {'value': self.sha256}
file_object['malware-sample'] = {'value': '{}|{}'.format(self.filename, self.md5), 'data': self.pseudofile}
# file_object['authentihash'] = self.
# file_object['sha-224'] = self.
# file_object['sha-384'] = self.
# file_object['sha512/224'] = self.
# file_object['sha512/256'] = self.
# file_object['tlsh'] = self.
return self._fill_object(file_object)

View File

@ -0,0 +1,102 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from pymisp import MISPEvent, MISPAttribute
import os
import json
import uuid
import abc
import sys
class MISPObjectException(Exception):
pass
class InvalidMISPObject(MISPObjectException):
"""Exception raised when an object doesn't contains the required field(s)"""
pass
class MISPObjectGenerator(metaclass=abc.ABCMeta):
def __init__(self, template_dir):
"""This class is used to fill a new MISP object with the default values defined in the object template
* template is the path to the template within the misp-object repository
* misp_objects_path is the path to the misp-object repository
"""
self.misp_objects_path = os.path.join(
os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)),
'data', 'misp-objects', 'objects')
with open(os.path.join(self.misp_objects_path, template_dir, 'definition.json'), 'r') as f:
self.definition = json.load(f)
self.misp_event = MISPEvent()
self.uuid = str(uuid.uuid4())
self.links = []
def _fill_object(self, values, strict=True):
"""Create a new object with the values gathered by the sub-class, use the default values from the template if needed"""
if strict:
self._validate(values)
# Create an empty object based om the object definition
new_object = self.__new_empty_object(self.definition)
if self.links:
# Set the links to other objects
new_object["ObjectReference"] = []
for link in self.links:
uuid, comment = link
new_object['ObjectReference'].append({'referenced_object_uuid': uuid, 'comment': comment})
for object_type, value in values.items():
# Add all the values as MISPAttributes to the current object
if value.get('value') is None:
continue
# Initialize the new MISPAttribute
attribute = MISPAttribute(self.misp_event.describe_types)
# Get the misp attribute type from the definition
value['type'] = self.definition['attributes'][object_type]['misp-attribute']
if value.get('disable_correlation') is None:
# The correlation can be disabled by default in the object definition.
# Use this value if it isn't overloaded by the object
value['disable_correlation'] = self.definition['attributes'][object_type].get('disable_correlation')
if value.get('to_ids') is None:
# Same for the to_ids flag
value['to_ids'] = self.definition['attributes'][object_type].get('to_ids')
# Set all the values in the MISP attribute
attribute.set_all_values(**value)
# Finalize the actual MISP Object
new_object['ObjectAttribute'].append({'type': object_type, 'Attribute': attribute._json()})
return new_object
def _validate(self, dump):
"""Make sure the object we're creating has the required fields"""
all_attribute_names = set(dump.keys())
if self.definition.get('requiredOneOf'):
if not set(self.definition['requiredOneOf']) & all_attribute_names:
raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self.definition['requiredOneOf'])))
if self.definition.get('required'):
for r in self.definition.get('required'):
if r not in all_attribute_names:
raise InvalidMISPObject('{} is required is required'.format(r))
return True
def add_link(self, uuid, comment=None):
"""Add a link (uuid) to an other object"""
self.links.append((uuid, comment))
def __new_empty_object(self, object_definiton):
"""Create a new empty object out of the template"""
return {'name': object_definiton['name'], 'meta-category': object_definiton['meta-category'],
'uuid': self.uuid, 'description': object_definiton['description'],
'version': object_definiton['version'], 'ObjectAttribute': []}
@abc.abstractmethod
def generate_attributes(self):
"""Contains the logic where all the values of the object are gathered"""
pass
@abc.abstractmethod
def dump(self):
"""This method normalize the attributes to add to the object.
It returns an python dictionary where the key is the type defined in the object,
and the value the value of the MISP Attribute"""
pass

172
pymisp/tools/peobject.py Normal file
View File

@ -0,0 +1,172 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from pymisp.tools import MISPObjectGenerator
from io import BytesIO
from hashlib import md5, sha1, sha256, sha512
from datetime import datetime
try:
import lief
HAS_LIEF = True
except ImportError:
HAS_LIEF = False
try:
import pydeep
HAS_PYDEEP = True
except ImportError:
HAS_PYDEEP = False
class PEObject(MISPObjectGenerator):
def __init__(self, parsed=None, filepath=None, pseudofile=None):
if not HAS_PYDEEP:
raise ImportError("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git")
if not HAS_LIEF:
raise ImportError('Please install lief, documentation here: https://github.com/lief-project/LIEF')
if pseudofile:
if isinstance(pseudofile, BytesIO):
self.pe = lief.PE.parse(raw=pseudofile.getvalue())
elif isinstance(pseudofile, bytes):
self.pe = lief.PE.parse(raw=pseudofile)
else:
raise Exception('Pseudo file can be BytesIO or bytes got {}'.format(type(pseudofile)))
elif filepath:
self.pe = lief.PE.parse(filepath)
elif parsed:
# Got an already parsed blob
if isinstance(parsed, lief.PE.Binary):
self.pe = parsed
else:
raise Exception('Not a lief.PE.Binary: {}'.format(type(parsed)))
MISPObjectGenerator.__init__(self, 'pe')
self.generate_attributes()
def _is_exe(self):
if not self._is_dll() and not self._is_driver():
return self.pe.header.has_characteristic(lief.PE.HEADER_CHARACTERISTICS.EXECUTABLE_IMAGE)
return False
def _is_dll(self):
return self.pe.header.has_characteristic(lief.PE.HEADER_CHARACTERISTICS.DLL)
def _is_driver(self):
# List from pefile
system_DLLs = set(('ntoskrnl.exe', 'hal.dll', 'ndis.sys', 'bootvid.dll', 'kdcom.dll'))
if system_DLLs.intersection([imp.lower() for imp in self.pe.libraries]):
return True
return False
def generate_attributes(self):
if self._is_dll():
self.pe_type = 'dll'
elif self._is_driver():
self.pe_type = 'driver'
elif self._is_exe():
self.pe_type = 'exe'
else:
self.pe_type = 'unknown'
# General information
self.entrypoint_address = self.pe.entrypoint
self.compilation_timestamp = datetime.utcfromtimestamp(self.pe.header.time_date_stamps).isoformat()
# self.imphash = self.pe.get_imphash()
try:
if (self.pe.has_resources and
self.pe.resources_manager.has_version and
self.pe.resources_manager.version.has_string_file_info and
self.pe.resources_manager.version.string_file_info.langcode_items):
fileinfo = dict(self.pe.resources_manager.version.string_file_info.langcode_items[0].items.items())
self.original_filename = fileinfo.get('OriginalFilename')
self.internal_filename = fileinfo.get('InternalName')
self.file_description = fileinfo.get('FileDescription')
self.file_version = fileinfo.get('FileVersion')
self.lang_id = self.pe.resources_manager.version.string_file_info.langcode_items[0].key
self.product_name = fileinfo.get('ProductName')
self.product_version = fileinfo.get('ProductVersion')
self.company_name = fileinfo.get('CompanyName')
self.legal_copyright = fileinfo.get('LegalCopyright')
except lief.read_out_of_bound:
# The file is corrupted
pass
# Sections
self.sections = []
if self.pe.sections:
pos = 0
for section in self.pe.sections:
s = PESectionObject(section)
self.add_link(s.uuid, 'Section {} of PE'.format(pos))
if ((self.entrypoint_address >= section.virtual_address) and
(self.entrypoint_address < (section.virtual_address + section.virtual_size))):
self.entrypoint_section = (section.name, pos) # Tuple: (section_name, position)
pos += 1
self.sections.append(s)
self.nb_sections = len(self.sections)
# TODO: TLSSection / DIRECTORY_ENTRY_TLS
def dump(self):
pe_object = {}
pe_object['type'] = {'value': self.pe_type}
if hasattr(self, 'imphash'):
pe_object['imphash'] = {'value': self.imphash}
if hasattr(self, 'original_filename'):
pe_object['original-filename'] = {'value': self.original_filename}
if hasattr(self, 'internal_filename'):
pe_object['internal-filename'] = {'value': self.internal_filename}
if hasattr(self, 'compilation_timestamp'):
pe_object['compilation-timestamp'] = {'value': self.compilation_timestamp}
if hasattr(self, 'entrypoint_section'):
pe_object['entrypoint-section|position'] = {'value': '{}|{}'.format(*self.entrypoint_section)}
if hasattr(self, 'entrypoint_address'):
pe_object['entrypoint-address'] = {'value': self.entrypoint_address}
if hasattr(self, 'file_description'):
pe_object['file-description'] = {'value': self.file_description}
if hasattr(self, 'file_version'):
pe_object['file-version'] = {'value': self.file_version}
if hasattr(self, 'lang_id'):
pe_object['lang-id'] = {'value': self.lang_id}
if hasattr(self, 'product_name'):
pe_object['product-name'] = {'value': self.product_name}
if hasattr(self, 'product_version'):
pe_object['product-version'] = {'value': self.product_version}
if hasattr(self, 'company_name'):
pe_object['company-name'] = {'value': self.company_name}
if hasattr(self, 'nb_sections'):
pe_object['number-sections'] = {'value': self.nb_sections}
return self._fill_object(pe_object)
class PESectionObject(MISPObjectGenerator):
def __init__(self, section):
MISPObjectGenerator.__init__(self, 'pe-section')
self.section = section
self.data = bytes(self.section.content)
self.generate_attributes()
def generate_attributes(self):
self.name = self.section.name
self.size = self.section.size
if self.size > 0:
self.entropy = self.section.entropy
self.md5 = md5(self.data).hexdigest()
self.sha1 = sha1(self.data).hexdigest()
self.sha256 = sha256(self.data).hexdigest()
self.sha512 = sha512(self.data).hexdigest()
if HAS_PYDEEP:
self.ssdeep = pydeep.hash_buf(self.data).decode()
def dump(self):
section = {}
section['name'] = {'value': self.name}
section['size-in-bytes'] = {'value': self.size}
if self.size > 0:
section['entropy'] = {'value': self.entropy}
section['md5'] = {'value': self.md5}
section['sha1'] = {'value': self.sha1}
section['sha256'] = {'value': self.sha256}
section['sha512'] = {'value': self.sha512}
section['ssdeep'] = {'value': self.ssdeep}
return self._fill_object(section)

View File

@ -0,0 +1,51 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from pymisp import PyMISP
from pymisp.tools import FileObject, PEObject
from pymisp.tools import make_binary_objects
import traceback
try:
import lief
HAS_LIEF = True
except ImportError:
HAS_LIEF = False
raise ImportError("Please install lief: https://github.com/lief-project/LIEF")
if __name__ == '__main__':
pymisp = PyMISP('https://mispbeta.circl.lu', 'et9ZEgn70YJ6URkCr6741LpJNAVUMYD1rM063od3')
# fo, peo, seos = make_objects('/home/raphael/.viper/projects/troopers17/vt_samples/1189/566ab945f61be016bfd9e83cc1b64f783b9b8deb891e6d504d3442bc8281b092')
import glob
for f in glob.glob('/home/raphael/.viper/projects/troopers17/vt_samples/*/*'):
#for f in glob.glob('/home/raphael/gits/pefile-tests/tests/corkami/*/*.exe'):
#for f in glob.glob('/home/raphael/gits/pefile-tests/tests/corkami/pocs/version_mini.exe'):
#for f in glob.glob('/home/raphael/gits/pefile-tests/tests/corkami/pocs/version_cust.exe'):
#for f in glob.glob('/home/raphael/gits/pefile-tests/tests/data/*.dll'):
print('\n', f)
try:
fo, peo, seos = make_binary_objects(f)
except Exception as e:
traceback.print_exc()
continue
continue
if fo:
response = pymisp.add_object(2221, 7, fo)
print(response)
if peo:
pymisp.add_object(2221, 11, peo)
if seos:
for s in seos:
pymisp.add_object(2221, 12, s)
#with open('fileobj.json', 'w') as f:
# json.dump(fo, f)
#with open('peobj.json', 'w') as f:
# json.dump(peo, f)
#with open('seobj.json', 'w') as f:
# json.dump(seos, f)
break

View File

@ -1,4 +1,4 @@
#!/usr/bin/python
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from setuptools import setup
import pymisp
@ -29,5 +29,6 @@ setup(
test_suite="tests",
install_requires=['requests', 'python-dateutil', 'jsonschema'],
include_package_data=True,
package_data={'data': ['schema.json', 'schema-lax.json', 'describeTypes.json']},
package_data={'pymisp': ['data/*.json', 'data/misp-objects/schema.json',
'data/misp-objects/objects/*/definition.json']},
)