MISP
/
PyMISP
mirror of https://github.com/MISP/PyMISP
2
0
Fork 0
Code Issues Releases Wiki Activity

Update file/pe/pe-sections objects creator.

pull/111/head
Raphaël Vinot 2017-08-23 15:36:13 +02:00
parent 1d6c63c54c
commit 77845bd813
11 changed files with 191 additions and 174 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
examples/add_file_object.py
View File

@ -7,7 +7,7 @@ import traceback
from keys import misp_url, misp_key, misp_verifycert
import glob
import argparse
import json
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Extract indicators out of binaries and add MISP objects to a MISP instance.')
@ -18,31 +18,27 @@ if __name__ == '__main__':
pymisp = PyMISP(misp_url, misp_key, misp_verifycert)
for f in glob.glob(args.path):
print('\n', f)
try:
fo, peo, seos = make_binary_objects(f)
except Exception as e:
traceback.print_exc()
continue
if fo:
template_id = pymisp.get_object_template_id(fo['name'])
try:
response = pymisp.add_object(args.event, template_id, fo)
print(response)
except Exception as e:
traceback.print_exc()
continue
continue
if peo:
template_id = pymisp.get_object_template_id(peo['name'])
print(template_id)
r = pymisp.add_object(args.event, template_id, peo)
print(r)
continue
if seos:
for s in seos:
print(s)
template_id = pymisp.get_object_template_id(s['name'])
r = pymisp.add_object(args.event, template_id, s)
print(r)
break
obj, refs = s
template_id = pymisp.get_object_template_id(obj['name'])
r = pymisp.add_object(args.event, template_id, obj)
if peo:
obj, refs = peo
template_id = pymisp.get_object_template_id(obj['name'])
r = pymisp.add_object(args.event, template_id, obj)
for ref in refs:
r = pymisp.add_object_reference(obj['uuid'], ref)
if fo:
obj, refs = fo
template_id = pymisp.get_object_template_id(obj['name'])
response = pymisp.add_object(args.event, template_id, obj)
for ref in refs:
r = pymisp.add_object_reference(obj['uuid'], ref)

1
pymisp/__init__.py
View File

@ -2,6 +2,7 @@ __version__ = '2.4.77'
from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey
from .api import PyMISP
from .abstract import AbstractMISP
from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull
from .tools import Neo4j
from .tools import stix

70
pymisp/abstract.py Normal file
View File

@ -0,0 +1,70 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import six # Remove that import when discarding python2 support.
import abc
import json
import collections
@six.add_metaclass(abc.ABCMeta) # Remove that line when discarding python2 support.
# Python3 way: class MISPObjectGenerator(metaclass=abc.ABCMeta):
class AbstractMISP(collections.MutableMapping):
attributes = None
def __init__(self):
"""Initialize the list of class-level attributes to set in the JSON dump"""
# The attribute names will be set automatically by the schemas when we will have them.
if self.attributes is None:
raise NotImplementedError('{} must define attributes'.format(type(self).__name__))
self.attributes = sorted(self.attributes)
def __check_dict_key(self, key):
if key not in self.attributes:
raise Exception('{} not a valid key in {}. Alowed keys: {}'.format(
key, type(self).__name__, ', '.join(self.attributes)))
return True
def from_dict(self, **kwargs):
for attribute in self.attributes:
val = kwargs.pop(attribute, None)
if val is None:
continue
setattr(self, attribute, val)
if kwargs:
raise Exception('Unused parameter(s): {}'.format(', '.join(kwargs.keys())))
def from_json(self, json_string):
"""Load a JSON string"""
self.from_dict(json.loads(json_string))
def to_dict(self):
to_return = {}
for attribute in self.attributes:
val = getattr(self, attribute, None)
if val is None:
continue
to_return[attribute] = val
return to_return
def to_json(self):
return json.dumps(self.to_dict(), indent=4, sort_keys=True)
def __getitem__(self, key):
if self.__check_dict_key(key):
return getattr(self, key)
def __setitem__(self, key, value):
if self.__check_dict_key(key):
setattr(self, key, value)
def __delitem__(self, key):
if self.__check_dict_key(key):
delattr(self, key)
def __iter__(self):
return iter(self.to_dict())
def __len__(self):
return len(self.to_dict())

6
pymisp/api.py
View File

@ -1588,6 +1588,12 @@ class PyMISP(object):
response = session.post(url, data=json.dumps(misp_object))
return self._check_response(response)
def add_object_reference(self, parent_uuid, misp_object_reference):
session = self.__prepare_session()
url = urljoin(self.root_url, 'object_references/add/{}'.format(parent_uuid))
response = session.post(url, data=json.dumps(misp_object_reference))
return self._check_response(response)
def get_object_templates_list(self):
session = self.__prepare_session()
url = urljoin(self.root_url, 'objectTemplates')

2
pymisp/data/misp-objects

@ -1 +1 @@
Subproject commit ca24684e2f49bcfbd886212ff003472716c26de9
Subproject commit 96d7aeb0729428a43f38f45b6c00a60b9fdba2b6

6
pymisp/mispevent.py
View File

@ -11,6 +11,7 @@ import base64
from io import BytesIO
from zipfile import ZipFile
import hashlib
from .abstract import AbstractMISP
try:
from dateutil.parser import parse
@ -198,8 +199,7 @@ class MISPAttribute(object):
self.malware_filename = self.value
m = hashlib.md5()
m.update(self.data.getvalue())
md5 = m.hexdigest()
self.value = '{}|{}'.format(self.malware_filename, md5)
self.value = self.malware_filename
self.malware_binary = self.data
self.encrypt = True
@ -280,7 +280,7 @@ def _int_to_str(d):
return d
class MISPEvent(object):
class MISPEvent(AbstractMISP):
def __init__(self, describe_types=None, strict_validation=False):
self.ressources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data')

2
pymisp/tools/create_misp_object.py
View File

@ -16,7 +16,7 @@ class FileTypeNotImplemented(MISPObjectException):
def make_pe_objects(lief_parsed, misp_file):
misp_pe = PEObject(parsed=lief_parsed)
misp_file.add_link(misp_pe.uuid, 'PE indicators')
misp_file.add_reference(misp_pe.uuid, 'included-in', 'PE indicators')
file_object = misp_file.dump()
pe_object = misp_pe.dump()
pe_sections = []

40
pymisp/tools/fileobject.py
View File

@ -45,15 +45,17 @@ class FileObject(MISPObjectGenerator):
self.generate_attributes()
def generate_attributes(self):
self.size = len(self.data)
if self.size > 0:
self.entropy = self.__entropy_H(self.data)
self.md5 = md5(self.data).hexdigest()
self.sha1 = sha1(self.data).hexdigest()
self.sha256 = sha256(self.data).hexdigest()
self.sha512 = sha512(self.data).hexdigest()
self.filetype = magic.from_buffer(self.data)
self.ssdeep = pydeep.hash_buf(self.data).decode()
self._create_attribute('filename', value=self.filename)
self._create_attribute('size-in-bytes', value=len(self.data))
if getattr(self, 'size-in-bytes').value > 0:
self._create_attribute('entropy', value=self.__entropy_H(self.data))
self._create_attribute('md5', value=md5(self.data).hexdigest())
self._create_attribute('sha1', value=sha1(self.data).hexdigest())
self._create_attribute('sha256', value=sha256(self.data).hexdigest())
self._create_attribute('sha512', value=sha512(self.data).hexdigest())
self._create_attribute('mimetype', value=magic.from_buffer(self.data))
self._create_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode())
self._create_attribute('malware-sample', value=self.filename.value, data=self.pseudofile)
def __entropy_H(self, data):
"""Calculate the entropy of a chunk of data."""
@ -70,23 +72,3 @@ class FileObject(MISPObjectGenerator):
entropy -= p_x * math.log(p_x, 2)
return entropy
def dump(self):
file_object = {}
file_object['filename'] = {'value': self.filename}
file_object['size-in-bytes'] = {'value': self.size}
if self.size > 0:
file_object['entropy'] = {'value': self.entropy}
file_object['ssdeep'] = {'value': self.ssdeep}
file_object['sha512'] = {'value': self.sha512}
file_object['md5'] = {'value': self.md5}
file_object['sha1'] = {'value': self.sha1}
file_object['sha256'] = {'value': self.sha256}
file_object['malware-sample'] = {'value': '{}|{}'.format(self.filename, self.md5), 'data': self.pseudofile}
# file_object['authentihash'] = self.
# file_object['sha-224'] = self.
# file_object['sha-384'] = self.
# file_object['sha512/224'] = self.
# file_object['sha512/256'] = self.
# file_object['tlsh'] = self.
return self._fill_object(file_object)

83
pymisp/tools/objectgenerator.py
View File

@ -1,7 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from pymisp import MISPEvent, MISPAttribute
from pymisp import MISPEvent, MISPAttribute, AbstractMISP
import os
import json
import uuid
@ -24,9 +24,17 @@ if six.PY2:
warnings.warn("You're using python 2, it is strongly recommended to use python >=3.4")
@six.add_metaclass(abc.ABCMeta) # Remove that line when discarding python2 support.
# Python3 way: class MISPObjectGenerator(metaclass=abc.ABCMeta):
class MISPObjectGenerator():
class MISPObjectReference(AbstractMISP):
attributes = ['uuid', 'relationship_type', 'comment']
def __init__(self, uuid, relationship_type, comment=None):
self['uuid'] = uuid
self['relationship_type'] = relationship_type
self['comment'] = comment
class MISPObjectGenerator(AbstractMISP):
def __init__(self, template_dir):
"""This class is used to fill a new MISP object with the default values defined in the object template
@ -38,46 +46,46 @@ class MISPObjectGenerator():
'data', 'misp-objects', 'objects')
with open(os.path.join(self.misp_objects_path, template_dir, 'definition.json'), 'r') as f:
self.definition = json.load(f)
self.attributes = self.definition['attributes'].keys()
self.misp_event = MISPEvent()
self.uuid = str(uuid.uuid4())
self.links = []
self.references = []
def _fill_object(self, values, strict=True):
def _create_attribute(self, object_type, **value):
if value.get('value') is None:
return None
# Initialize the new MISPAttribute
# Get the misp attribute type from the definition
value['type'] = self.definition['attributes'][object_type]['misp-attribute']
if value.get('disable_correlation') is None:
# The correlation can be disabled by default in the object definition.
# Use this value if it isn't overloaded by the object
value['disable_correlation'] = self.definition['attributes'][object_type].get('disable_correlation')
if value.get('to_ids') is None:
# Same for the to_ids flag
value['to_ids'] = self.definition['attributes'][object_type].get('to_ids')
# Set all the values in the MISP attribute
attribute = MISPAttribute(self.misp_event.describe_types)
attribute.set_all_values(**value)
self[object_type] = attribute
def dump(self, strict=True):
"""Create a new object with the values gathered by the sub-class, use the default values from the template if needed"""
if strict:
self._validate(values)
self._validate()
# Create an empty object based om the object definition
new_object = self.__new_empty_object(self.definition)
if self.links:
# Set the links to other objects
new_object["ObjectReference"] = []
for link in self.links:
uuid, comment = link
new_object['ObjectReference'].append({'referenced_object_uuid': uuid, 'comment': comment})
for object_type, value in values.items():
for object_type, attribute in self.items():
# Add all the values as MISPAttributes to the current object
if value.get('value') is None:
if attribute.value is None:
continue
# Initialize the new MISPAttribute
attribute = MISPAttribute(self.misp_event.describe_types)
# Get the misp attribute type from the definition
value['type'] = self.definition['attributes'][object_type]['misp-attribute']
if value.get('disable_correlation') is None:
# The correlation can be disabled by default in the object definition.
# Use this value if it isn't overloaded by the object
value['disable_correlation'] = self.definition['attributes'][object_type].get('disable_correlation')
if value.get('to_ids') is None:
# Same for the to_ids flag
value['to_ids'] = self.definition['attributes'][object_type].get('to_ids')
# Set all the values in the MISP attribute
attribute.set_all_values(**value)
# Finalize the actual MISP Object
new_object['Attribute'].append({'type': object_type, 'Attribute': attribute._json()})
return new_object
new_object['Attribute'].append({'object_relation': object_type, **attribute._json()})
return new_object, [r.to_dict() for r in self.references]
def _validate(self, dump):
def _validate(self):
"""Make sure the object we're creating has the required fields"""
all_attribute_names = set(dump.keys())
all_attribute_names = set(self.keys())
if self.definition.get('requiredOneOf'):
if not set(self.definition['requiredOneOf']) & all_attribute_names:
raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self.definition['requiredOneOf'])))
@ -87,9 +95,9 @@ class MISPObjectGenerator():
raise InvalidMISPObject('{} is required is required'.format(r))
return True
def add_link(self, uuid, comment=None):
def add_reference(self, uuid, relationship_type, comment=None):
"""Add a link (uuid) to an other object"""
self.links.append((uuid, comment))
self.references.append(MISPObjectReference(uuid, relationship_type, comment))
def __new_empty_object(self, object_definiton):
"""Create a new empty object out of the template"""
@ -101,10 +109,3 @@ class MISPObjectGenerator():
def generate_attributes(self):
"""Contains the logic where all the values of the object are gathered"""
pass
@abc.abstractmethod
def dump(self):
"""This method normalize the attributes to add to the object.
It returns an python dictionary where the key is the type defined in the object,
and the value the value of the MISP Attribute"""
pass

107
pymisp/tools/peobject.py
View File

@ -60,18 +60,21 @@ class PEObject(MISPObjectGenerator):
return True
return False
def generate_attributes(self):
def _get_pe_type(self):
if self._is_dll():
self.pe_type = 'dll'
return 'dll'
elif self._is_driver():
self.pe_type = 'driver'
return 'driver'
elif self._is_exe():
self.pe_type = 'exe'
return 'exe'
else:
self.pe_type = 'unknown'
return 'unknown'
def generate_attributes(self):
self._create_attribute('type', value=self._get_pe_type())
# General information
self.entrypoint_address = self.pe.entrypoint
self.compilation_timestamp = datetime.utcfromtimestamp(self.pe.header.time_date_stamps).isoformat()
self._create_attribute('entrypoint-address', value=self.pe.entrypoint)
self._create_attribute('compilation-timestamp', value=datetime.utcfromtimestamp(self.pe.header.time_date_stamps).isoformat())
# self.imphash = self.pe.get_imphash()
try:
if (self.pe.has_resources and
@ -79,15 +82,15 @@ class PEObject(MISPObjectGenerator):
self.pe.resources_manager.version.has_string_file_info and
self.pe.resources_manager.version.string_file_info.langcode_items):
fileinfo = dict(self.pe.resources_manager.version.string_file_info.langcode_items[0].items.items())
self.original_filename = fileinfo.get('OriginalFilename')
self.internal_filename = fileinfo.get('InternalName')
self.file_description = fileinfo.get('FileDescription')
self.file_version = fileinfo.get('FileVersion')
self.lang_id = self.pe.resources_manager.version.string_file_info.langcode_items[0].key
self.product_name = fileinfo.get('ProductName')
self.product_version = fileinfo.get('ProductVersion')
self.company_name = fileinfo.get('CompanyName')
self.legal_copyright = fileinfo.get('LegalCopyright')
self._create_attribute('original-filename', value=fileinfo.get('OriginalFilename'))
self._create_attribute('internal-filename', value=fileinfo.get('InternalName'))
self._create_attribute('file-description', value=fileinfo.get('FileDescription'))
self._create_attribute('file-version', value=fileinfo.get('FileVersion'))
self._create_attribute('lang-id', value=self.pe.resources_manager.version.string_file_info.langcode_items[0].key)
self._create_attribute('product-name', value=fileinfo.get('ProductName'))
self._create_attribute('product-version', value=fileinfo.get('ProductVersion'))
self._create_attribute('company-name', value=fileinfo.get('CompanyName'))
self._create_attribute('legal-copyright', value=fileinfo.get('LegalCopyright'))
except lief.read_out_of_bound:
# The file is corrupted
pass
@ -97,46 +100,15 @@ class PEObject(MISPObjectGenerator):
pos = 0
for section in self.pe.sections:
s = PESectionObject(section)
self.add_link(s.uuid, 'Section {} of PE'.format(pos))
if ((self.entrypoint_address >= section.virtual_address) and
(self.entrypoint_address < (section.virtual_address + section.virtual_size))):
self.entrypoint_section = (section.name, pos) # Tuple: (section_name, position)
self.add_reference(s.uuid, 'included-in', 'Section {} of PE'.format(pos))
if ((self.pe.entrypoint >= section.virtual_address) and
(self.pe.entrypoint < (section.virtual_address + section.virtual_size))):
self._create_attribute('entrypoint-section|position', value='{}|{}'.format(section.name, pos))
pos += 1
self.sections.append(s)
self.nb_sections = len(self.sections)
self._create_attribute('number-sections', value=len(self.sections))
# TODO: TLSSection / DIRECTORY_ENTRY_TLS
def dump(self):
pe_object = {}
pe_object['type'] = {'value': self.pe_type}
if hasattr(self, 'imphash'):
pe_object['imphash'] = {'value': self.imphash}
if hasattr(self, 'original_filename'):
pe_object['original-filename'] = {'value': self.original_filename}
if hasattr(self, 'internal_filename'):
pe_object['internal-filename'] = {'value': self.internal_filename}
if hasattr(self, 'compilation_timestamp'):
pe_object['compilation-timestamp'] = {'value': self.compilation_timestamp}
if hasattr(self, 'entrypoint_section'):
pe_object['entrypoint-section|position'] = {'value': '{}|{}'.format(*self.entrypoint_section)}
if hasattr(self, 'entrypoint_address'):
pe_object['entrypoint-address'] = {'value': self.entrypoint_address}
if hasattr(self, 'file_description'):
pe_object['file-description'] = {'value': self.file_description}
if hasattr(self, 'file_version'):
pe_object['file-version'] = {'value': self.file_version}
if hasattr(self, 'lang_id'):
pe_object['lang-id'] = {'value': self.lang_id}
if hasattr(self, 'product_name'):
pe_object['product-name'] = {'value': self.product_name}
if hasattr(self, 'product_version'):
pe_object['product-version'] = {'value': self.product_version}
if hasattr(self, 'company_name'):
pe_object['company-name'] = {'value': self.company_name}
if hasattr(self, 'nb_sections'):
pe_object['number-sections'] = {'value': self.nb_sections}
return self._fill_object(pe_object)
class PESectionObject(MISPObjectGenerator):
@ -147,26 +119,13 @@ class PESectionObject(MISPObjectGenerator):
self.generate_attributes()
def generate_attributes(self):
self.name = self.section.name
self.size = self.section.size
if self.size > 0:
self.entropy = self.section.entropy
self.md5 = md5(self.data).hexdigest()
self.sha1 = sha1(self.data).hexdigest()
self.sha256 = sha256(self.data).hexdigest()
self.sha512 = sha512(self.data).hexdigest()
self._create_attribute('name', value=self.section.name)
self._create_attribute('size-in-bytes', value=self.section.size)
if getattr(self, 'size-in-bytes').value > 0:
self._create_attribute('entropy', value=self.section.entropy)
self._create_attribute('md5', value=md5(self.data).hexdigest())
self._create_attribute('sha1', value=sha1(self.data).hexdigest())
self._create_attribute('sha256', value=sha256(self.data).hexdigest())
self._create_attribute('sha512', value=sha512(self.data).hexdigest())
if HAS_PYDEEP:
self.ssdeep = pydeep.hash_buf(self.data).decode()
def dump(self):
section = {}
section['name'] = {'value': self.name}
section['size-in-bytes'] = {'value': self.size}
if self.size > 0:
section['entropy'] = {'value': self.entropy}
section['md5'] = {'value': self.md5}
section['sha1'] = {'value': self.sha1}
section['sha256'] = {'value': self.sha256}
section['sha512'] = {'value': self.sha512}
section['ssdeep'] = {'value': self.ssdeep}
return self._fill_object(section)
self._create_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode())

6
setup.py
View File

@ -29,6 +29,8 @@ setup(
test_suite="tests",
install_requires=['requests', 'python-dateutil', 'jsonschema'],
include_package_data=True,
package_data={'pymisp': ['data/*.json', 'data/misp-objects/schema.json',
'data/misp-objects/objects/*/definition.json']},
package_data={'pymisp': ['data/*.json', 'data/misp-objects/schema_objects.json',
'data/misp-objects/schema_relationships.json',
'data/misp-objects/objects/*/definition.json',
'data/misp-objects/relationships/definition.json']},
)