mirror of https://github.com/MISP/PyMISP
Refactoring in order to load objects
Raphaël Vinot
parent
44f32bc443
commit
2bc0745fbf
|
|
@ -30,11 +30,11 @@ if __name__ == '__main__':
|
|||
if peo:
|
||||
template_id = pymisp.get_object_template_id(peo.template_uuid)
|
||||
r = pymisp.add_object(args.event, template_id, peo)
|
||||
for ref in peo.references:
|
||||
for ref in peo.ObjectReference:
|
||||
r = pymisp.add_object_reference(ref)
|
||||
|
||||
if fo:
|
||||
template_id = pymisp.get_object_template_id(fo.template_uuid)
|
||||
response = pymisp.add_object(args.event, template_id, fo)
|
||||
for ref in fo.references:
|
||||
for ref in fo.ObjectReference:
|
||||
r = pymisp.add_object_reference(ref)
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ try:
|
|||
except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
def check():
|
||||
missing_dependencies = {'pydeep': False, 'lief': False, 'magic': False, 'pymisp': False}
|
||||
try:
|
||||
|
|
@ -38,18 +39,18 @@ def make_objects(path):
|
|||
if seos:
|
||||
for s in seos:
|
||||
to_return['objects'].append(s)
|
||||
if s.references:
|
||||
to_return['references'] += s.references
|
||||
if s.ObjectReference:
|
||||
to_return['references'] += s.ObjectReference
|
||||
|
||||
if peo:
|
||||
to_return['objects'].append(peo)
|
||||
if peo.references:
|
||||
to_return['references'] += peo.references
|
||||
if peo.ObjectReference:
|
||||
to_return['references'] += peo.ObjectReference
|
||||
|
||||
if fo:
|
||||
to_return['objects'].append(fo)
|
||||
if fo.references:
|
||||
to_return['references'] += fo.references
|
||||
if fo.ObjectReference:
|
||||
to_return['references'] += fo.ObjectReference
|
||||
return json.dumps(to_return, cls=MISPEncode)
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
__version__ = '2.4.77'
|
||||
|
||||
try:
|
||||
from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey
|
||||
from .api import PyMISP
|
||||
from .abstract import AbstractMISP, MISPEncode
|
||||
from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull
|
||||
from .tools import Neo4j
|
||||
from .tools import stix
|
||||
from .tools import MISPObjectGenerator
|
||||
from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate # noqa
|
||||
from .api import PyMISP # noqa
|
||||
from .abstract import AbstractMISP, MISPEncode # noqa
|
||||
from .defaultobjects import MISPObject, AbstractMISPObjectGenerator # noqa
|
||||
from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull # noqa
|
||||
from .tools import Neo4j # noqa
|
||||
from .tools import stix # noqa
|
||||
except ImportError:
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -5,9 +5,13 @@ import abc
|
|||
import json
|
||||
from json import JSONEncoder
|
||||
import collections
|
||||
|
||||
import six # Remove that import when discarding python2 support.
|
||||
|
||||
if six.PY2:
|
||||
import warnings
|
||||
warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5")
|
||||
|
||||
|
||||
class MISPEncode(JSONEncoder):
|
||||
|
||||
def default(self, obj):
|
||||
|
|
@ -17,7 +21,6 @@ class MISPEncode(JSONEncoder):
|
|||
|
||||
|
||||
@six.add_metaclass(abc.ABCMeta) # Remove that line when discarding python2 support.
|
||||
# Python3 way: class MISPObjectGenerator(metaclass=abc.ABCMeta):
|
||||
class AbstractMISP(collections.MutableMapping):
|
||||
|
||||
attributes = None
|
||||
|
|
|
|||
|
|
@ -0,0 +1,195 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import abc
|
||||
import os
|
||||
import json
|
||||
import sys
|
||||
import uuid
|
||||
from collections import Counter
|
||||
|
||||
from .abstract import AbstractMISP
|
||||
from .exceptions import UnknownMISPObjectTemplate, InvalidMISPObject
|
||||
import six # Remove that import when discarding python2 support.
|
||||
|
||||
|
||||
if six.PY2:
|
||||
import warnings
|
||||
warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5")
|
||||
|
||||
|
||||
class MISPObjectReference(AbstractMISP):
|
||||
|
||||
attributes = ['source_uuid', 'destination_uuid', 'relationship_type', 'comment', 'uuid', 'deleted']
|
||||
|
||||
def __init__(self):
|
||||
super(MISPObjectReference, self).__init__()
|
||||
|
||||
def from_dict(self, source_uuid, destination_uuid, relationship_type, comment=None, **kwargs):
|
||||
self.source_uuid = source_uuid
|
||||
self.destination_uuid = destination_uuid
|
||||
self.relationship_type = relationship_type
|
||||
self.comment = comment
|
||||
for k, v in kwargs:
|
||||
setattr(self, k, v)
|
||||
|
||||
|
||||
class MISPObjectAttribute(AbstractMISP):
|
||||
|
||||
# This list is very limited and hardcoded to fit the current needs (file/pe/pesection creation): MISPAttriute will follow the
|
||||
# same spec and just add one attribute: object_relation
|
||||
attributes = ['object_relation', 'value', 'type', 'category', 'disable_correlation', 'to_ids',
|
||||
'data', 'encrypt', 'distribution', 'comment', 'uuid', 'event_id']
|
||||
|
||||
def __init__(self, definition):
|
||||
super(MISPObjectAttribute, self).__init__()
|
||||
self.definition = definition
|
||||
|
||||
def from_dict(self, object_relation, value, **kwargs):
|
||||
from .mispevent import MISPAttribute
|
||||
self.object_relation = object_relation
|
||||
self.value = value
|
||||
# Initialize the new MISPAttribute
|
||||
# Get the misp attribute type from the definition
|
||||
self.type = kwargs.pop('type', None)
|
||||
if self.type is None:
|
||||
self.type = self.definition.get('misp-attribute')
|
||||
self.disable_correlation = kwargs.pop('disable_correlation', None)
|
||||
if self.disable_correlation is None:
|
||||
# The correlation can be disabled by default in the object definition.
|
||||
# Use this value if it isn't overloaded by the object
|
||||
self.disable_correlation = self.definition.get('disable_correlation')
|
||||
self.to_ids = kwargs.pop('to_ids', None)
|
||||
if self.to_ids is None:
|
||||
# Same for the to_ids flag
|
||||
self.to_ids = self.definition.get('to_ids')
|
||||
# FIXME: dirty hack until all the classes are ported to the new format but we get the default values
|
||||
# Initialise rest of the values
|
||||
for k, v in kwargs.items():
|
||||
setattr(self, k, v)
|
||||
temp_attribute = MISPAttribute()
|
||||
temp_attribute.set_all_values(**self)
|
||||
# Update default values
|
||||
for k, v in temp_attribute.to_dict().items():
|
||||
setattr(self, k, v)
|
||||
|
||||
|
||||
class MISPObject(AbstractMISP):
|
||||
|
||||
attributes = ['name', 'meta-category', 'uuid', 'description', 'template_version', 'template_uuid', 'Attribute']
|
||||
|
||||
def __init__(self, name, strict=True):
|
||||
super(MISPObject, self).__init__()
|
||||
self.strict = strict
|
||||
self.name = name
|
||||
self.misp_objects_path = os.path.join(
|
||||
os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)),
|
||||
'data', 'misp-objects', 'objects')
|
||||
if os.path.exists(os.path.join(self.misp_objects_path, self.name, 'definition.json')):
|
||||
self.known_template = True
|
||||
else:
|
||||
if self.strict:
|
||||
raise UnknownMISPObjectTemplate('{} is unknown in the MISP object directory.')
|
||||
else:
|
||||
self.known_template = False
|
||||
if self.known_template:
|
||||
with open(os.path.join(self.misp_objects_path, self.name, 'definition.json'), 'r') as f:
|
||||
self.definition = json.load(f)
|
||||
setattr(self, 'meta-category', self.definition['meta-category'])
|
||||
self.template_uuid = self.definition['uuid']
|
||||
self.description = self.definition['description']
|
||||
self.template_version = self.definition['version']
|
||||
else:
|
||||
# FIXME We need to set something for meta-category, template_uuid, description and template_version
|
||||
pass
|
||||
self.uuid = str(uuid.uuid4())
|
||||
self.Attribute = []
|
||||
self.ObjectReference = []
|
||||
|
||||
def from_dict(self, **kwargs):
|
||||
if self.known_template:
|
||||
if kwargs.get('template_uuid') and kwargs['template_uuid'] != self.template_uuid:
|
||||
if self.strict:
|
||||
raise UnknownMISPObjectTemplate('UUID of the object is different from the one of the template.')
|
||||
else:
|
||||
self.known_template = False
|
||||
if kwargs.get('template_version') and int(kwargs['template_version']) != self.template_version:
|
||||
if self.strict:
|
||||
raise UnknownMISPObjectTemplate('Version of the object ({}) is different from the one of the template ({}).'.format(kwargs['template_version'], self.template_version))
|
||||
else:
|
||||
self.known_template = False
|
||||
|
||||
for key, value in kwargs.items():
|
||||
if key == 'Attribute':
|
||||
for v in value:
|
||||
self.add_attribute(**v)
|
||||
elif key == 'ObjectReference':
|
||||
for v in value:
|
||||
self.add_reference(**v)
|
||||
else:
|
||||
setattr(self, key, value)
|
||||
|
||||
def to_dict(self, strict=True):
|
||||
if strict or self.strict and self.known_template:
|
||||
self._validate()
|
||||
return super(MISPObject, self).to_dict()
|
||||
|
||||
def to_json(self, strict=True):
|
||||
if strict or self.strict and self.known_template:
|
||||
self._validate()
|
||||
return super(MISPObject, self).to_json()
|
||||
|
||||
def _validate(self):
|
||||
"""Make sure the object we're creating has the required fields"""
|
||||
all_object_relations = []
|
||||
for a in self.Attribute:
|
||||
all_object_relations.append(a.object_relation)
|
||||
count_relations = dict(Counter(all_object_relations))
|
||||
for key, counter in count_relations.items():
|
||||
if counter == 1:
|
||||
continue
|
||||
if not self.definition['attributes'][key].get('multiple'):
|
||||
raise InvalidMISPObject('Multiple occurrences of {} is not allowed'.format(key))
|
||||
all_attribute_names = set(count_relations.keys())
|
||||
if self.definition.get('requiredOneOf'):
|
||||
if not set(self.definition['requiredOneOf']) & all_attribute_names:
|
||||
raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self.definition['requiredOneOf'])))
|
||||
if self.definition.get('required'):
|
||||
for r in self.definition.get('required'):
|
||||
if r not in all_attribute_names:
|
||||
raise InvalidMISPObject('{} is required'.format(r))
|
||||
return True
|
||||
|
||||
def add_reference(self, destination_uuid, relationship_type, comment=None, **kwargs):
|
||||
"""Add a link (uuid) to an other object"""
|
||||
if kwargs.get('source_uuid'):
|
||||
# Load existing object
|
||||
source_uuid = kwargs.get('source_uuid')
|
||||
else:
|
||||
# New reference
|
||||
source_uuid = self.uuid
|
||||
reference = MISPObjectReference()
|
||||
reference.from_dict(source_uuid=source_uuid, destination_uuid=destination_uuid,
|
||||
relationship_type=relationship_type, comment=comment, **kwargs)
|
||||
self.ObjectReference.append(reference)
|
||||
|
||||
def add_attribute(self, object_relation, **value):
|
||||
if value.get('value') is None:
|
||||
return None
|
||||
if self.known_template:
|
||||
attribute = MISPObjectAttribute(self.definition['attributes'][object_relation])
|
||||
else:
|
||||
attribute = MISPObjectAttribute({})
|
||||
attribute.from_dict(object_relation, **value)
|
||||
self.Attribute.append(attribute)
|
||||
return attribute
|
||||
|
||||
|
||||
@six.add_metaclass(abc.ABCMeta) # Remove that line when discarding python2 support.
|
||||
# Python3 way: class MISPObjectGenerator(metaclass=abc.ABCMeta):
|
||||
class AbstractMISPObjectGenerator(MISPObject):
|
||||
|
||||
@abc.abstractmethod
|
||||
def generate_attributes(self):
|
||||
"""Contains the logic where all the values of the object are gathered"""
|
||||
pass
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
|
||||
class PyMISPError(Exception):
|
||||
def __init__(self, message):
|
||||
super(PyMISPError, self).__init__(message)
|
||||
|
|
@ -29,3 +30,16 @@ class NoURL(PyMISPError):
|
|||
|
||||
class NoKey(PyMISPError):
|
||||
pass
|
||||
|
||||
|
||||
class MISPObjectException(PyMISPError):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidMISPObject(MISPObjectException):
|
||||
"""Exception raised when an object doesn't respect the contrains in the definition"""
|
||||
pass
|
||||
|
||||
class UnknownMISPObjectTemplate(MISPObjectException):
|
||||
"""Exception raised when the template is unknown"""
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ import base64
|
|||
from io import BytesIO
|
||||
from zipfile import ZipFile
|
||||
import hashlib
|
||||
from .abstract import AbstractMISP
|
||||
|
||||
try:
|
||||
from dateutil.parser import parse
|
||||
|
|
@ -38,6 +37,7 @@ except ImportError:
|
|||
has_pyme = False
|
||||
|
||||
from .exceptions import PyMISPError, NewEventError, NewAttributeError
|
||||
from .defaultobjects import MISPObject
|
||||
|
||||
# Least dirty way to support python 2 and 3
|
||||
try:
|
||||
|
|
@ -285,7 +285,7 @@ def _int_to_str(d):
|
|||
return d
|
||||
|
||||
|
||||
class MISPEvent(AbstractMISP):
|
||||
class MISPEvent(object):
|
||||
|
||||
def __init__(self, describe_types=None, strict_validation=False):
|
||||
self.ressources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data')
|
||||
|
|
@ -338,6 +338,7 @@ class MISPEvent(AbstractMISP):
|
|||
self.RelatedEvent = []
|
||||
self.Tag = []
|
||||
self.Galaxy = None
|
||||
self.Object = None
|
||||
|
||||
def _serialize(self):
|
||||
return '{date}{threat_level_id}{info}{uuid}{analysis}{timestamp}'.format(
|
||||
|
|
@ -510,6 +511,12 @@ class MISPEvent(AbstractMISP):
|
|||
self.sig = kwargs['sig']
|
||||
if kwargs.get('global_sig'):
|
||||
self.global_sig = kwargs['global_sig']
|
||||
if kwargs.get('Object'):
|
||||
self.Object = []
|
||||
for obj in kwargs['Object']:
|
||||
tmp_object = MISPObject(obj['name'])
|
||||
tmp_object.from_dict(**obj)
|
||||
self.Object.append(tmp_object)
|
||||
|
||||
def _json(self):
|
||||
# DEPTECATED
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
from .neo4j import Neo4j # noqa
|
||||
from .objectgenerator import MISPObjectGenerator, MISPObjectException, InvalidMISPObject # noqa
|
||||
from .fileobject import FileObject # noqa
|
||||
from .peobject import PEObject, PESectionObject # noqa
|
||||
from .elfobject import ELFObject, ELFSectionObject # noqa
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from pymisp.tools import FileObject, PEObject, ELFObject, MachOObject, MISPObjectException
|
||||
from pymisp.tools import FileObject, PEObject, ELFObject, MachOObject
|
||||
from pymisp.exceptions import MISPObjectException
|
||||
import warnings
|
||||
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from pymisp.tools import MISPObjectGenerator
|
||||
from pymisp.defaultobjects import AbstractMISPObjectGenerator
|
||||
from io import BytesIO
|
||||
from hashlib import md5, sha1, sha256, sha512
|
||||
import warnings
|
||||
|
|
@ -20,7 +20,7 @@ except ImportError:
|
|||
HAS_PYDEEP = False
|
||||
|
||||
|
||||
class ELFObject(MISPObjectGenerator):
|
||||
class ELFObject(AbstractMISPObjectGenerator):
|
||||
|
||||
def __init__(self, parsed=None, filepath=None, pseudofile=None):
|
||||
if not HAS_PYDEEP:
|
||||
|
|
@ -49,10 +49,10 @@ class ELFObject(MISPObjectGenerator):
|
|||
|
||||
def generate_attributes(self):
|
||||
# General information
|
||||
self._create_attribute('type', value=str(self.elf.header.file_type).split('.')[1])
|
||||
self._create_attribute('entrypoint-address', value=self.elf.entrypoint)
|
||||
self._create_attribute('arch', value=str(self.elf.header.machine_type).split('.')[1])
|
||||
self._create_attribute('os_abi', value=str(self.elf.header.identity_os_abi).split('.')[1])
|
||||
self.add_attribute('type', value=str(self.elf.header.file_type).split('.')[1])
|
||||
self.add_attribute('entrypoint-address', value=self.elf.entrypoint)
|
||||
self.add_attribute('arch', value=str(self.elf.header.machine_type).split('.')[1])
|
||||
self.add_attribute('os_abi', value=str(self.elf.header.identity_os_abi).split('.')[1])
|
||||
# Sections
|
||||
self.sections = []
|
||||
if self.elf.sections:
|
||||
|
|
@ -62,10 +62,10 @@ class ELFObject(MISPObjectGenerator):
|
|||
self.add_reference(s.uuid, 'included-in', 'Section {} of ELF'.format(pos))
|
||||
pos += 1
|
||||
self.sections.append(s)
|
||||
self._create_attribute('number-sections', value=len(self.sections))
|
||||
self.add_attribute('number-sections', value=len(self.sections))
|
||||
|
||||
|
||||
class ELFSectionObject(MISPObjectGenerator):
|
||||
class ELFSectionObject(AbstractMISPObjectGenerator):
|
||||
|
||||
def __init__(self, section):
|
||||
# Python3 way
|
||||
|
|
@ -76,16 +76,16 @@ class ELFSectionObject(MISPObjectGenerator):
|
|||
self.generate_attributes()
|
||||
|
||||
def generate_attributes(self):
|
||||
self._create_attribute('name', value=self.section.name)
|
||||
self._create_attribute('type', value=str(self.section.type).split('.')[1])
|
||||
self.add_attribute('name', value=self.section.name)
|
||||
self.add_attribute('type', value=str(self.section.type).split('.')[1])
|
||||
for flag in self.section.flags_list:
|
||||
self._create_attribute('flag', value=str(flag).split('.')[1])
|
||||
size = self._create_attribute('size-in-bytes', value=self.section.size)
|
||||
self.add_attribute('flag', value=str(flag).split('.')[1])
|
||||
size = self.add_attribute('size-in-bytes', value=self.section.size)
|
||||
if int(size.value) > 0:
|
||||
self._create_attribute('entropy', value=self.section.entropy)
|
||||
self._create_attribute('md5', value=md5(self.data).hexdigest())
|
||||
self._create_attribute('sha1', value=sha1(self.data).hexdigest())
|
||||
self._create_attribute('sha256', value=sha256(self.data).hexdigest())
|
||||
self._create_attribute('sha512', value=sha512(self.data).hexdigest())
|
||||
self.add_attribute('entropy', value=self.section.entropy)
|
||||
self.add_attribute('md5', value=md5(self.data).hexdigest())
|
||||
self.add_attribute('sha1', value=sha1(self.data).hexdigest())
|
||||
self.add_attribute('sha256', value=sha256(self.data).hexdigest())
|
||||
self.add_attribute('sha512', value=sha512(self.data).hexdigest())
|
||||
if HAS_PYDEEP:
|
||||
self._create_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode())
|
||||
self.add_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode())
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from pymisp.tools import MISPObjectGenerator
|
||||
from pymisp.defaultobjects import AbstractMISPObjectGenerator
|
||||
import os
|
||||
from io import BytesIO
|
||||
from hashlib import md5, sha1, sha256, sha512
|
||||
|
|
@ -22,7 +22,7 @@ except ImportError:
|
|||
HAS_MAGIC = False
|
||||
|
||||
|
||||
class FileObject(MISPObjectGenerator):
|
||||
class FileObject(AbstractMISPObjectGenerator):
|
||||
|
||||
def __init__(self, filepath=None, pseudofile=None, filename=None):
|
||||
if not HAS_PYDEEP:
|
||||
|
|
@ -48,19 +48,19 @@ class FileObject(MISPObjectGenerator):
|
|||
self.generate_attributes()
|
||||
|
||||
def generate_attributes(self):
|
||||
self._create_attribute('filename', value=self.filename)
|
||||
size = self._create_attribute('size-in-bytes', value=len(self.data))
|
||||
self.add_attribute('filename', value=self.filename)
|
||||
size = self.add_attribute('size-in-bytes', value=len(self.data))
|
||||
if int(size.value) > 0:
|
||||
self._create_attribute('entropy', value=self.__entropy_H(self.data))
|
||||
self._create_attribute('md5', value=md5(self.data).hexdigest())
|
||||
self._create_attribute('sha1', value=sha1(self.data).hexdigest())
|
||||
self._create_attribute('sha256', value=sha256(self.data).hexdigest())
|
||||
self._create_attribute('sha512', value=sha512(self.data).hexdigest())
|
||||
self._create_attribute('malware-sample', value=self.filename, data=self.pseudofile)
|
||||
self.add_attribute('entropy', value=self.__entropy_H(self.data))
|
||||
self.add_attribute('md5', value=md5(self.data).hexdigest())
|
||||
self.add_attribute('sha1', value=sha1(self.data).hexdigest())
|
||||
self.add_attribute('sha256', value=sha256(self.data).hexdigest())
|
||||
self.add_attribute('sha512', value=sha512(self.data).hexdigest())
|
||||
self.add_attribute('malware-sample', value=self.filename, data=self.pseudofile)
|
||||
if HAS_MAGIC:
|
||||
self._create_attribute('mimetype', value=magic.from_buffer(self.data))
|
||||
self.add_attribute('mimetype', value=magic.from_buffer(self.data))
|
||||
if HAS_PYDEEP:
|
||||
self._create_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode())
|
||||
self.add_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode())
|
||||
|
||||
def __entropy_H(self, data):
|
||||
"""Calculate the entropy of a chunk of data."""
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from pymisp.tools import MISPObjectGenerator
|
||||
from pymisp.defaultobjects import AbstractMISPObjectGenerator
|
||||
from io import BytesIO
|
||||
from hashlib import md5, sha1, sha256, sha512
|
||||
import warnings
|
||||
|
|
@ -20,7 +20,7 @@ except ImportError:
|
|||
HAS_PYDEEP = False
|
||||
|
||||
|
||||
class MachOObject(MISPObjectGenerator):
|
||||
class MachOObject(AbstractMISPObjectGenerator):
|
||||
|
||||
def __init__(self, parsed=None, filepath=None, pseudofile=None):
|
||||
if not HAS_PYDEEP:
|
||||
|
|
@ -48,11 +48,11 @@ class MachOObject(MISPObjectGenerator):
|
|||
self.generate_attributes()
|
||||
|
||||
def generate_attributes(self):
|
||||
self._create_attribute('type', value=str(self.macho.header.file_type).split('.')[1])
|
||||
self._create_attribute('name', value=self.macho.name)
|
||||
self.add_attribute('type', value=str(self.macho.header.file_type).split('.')[1])
|
||||
self.add_attribute('name', value=self.macho.name)
|
||||
# General information
|
||||
if self.macho.has_entrypoint:
|
||||
self._create_attribute('entrypoint-address', value=self.macho.entrypoint)
|
||||
self.add_attribute('entrypoint-address', value=self.macho.entrypoint)
|
||||
# Sections
|
||||
self.sections = []
|
||||
if self.macho.sections:
|
||||
|
|
@ -62,10 +62,10 @@ class MachOObject(MISPObjectGenerator):
|
|||
self.add_reference(s.uuid, 'included-in', 'Section {} of MachO'.format(pos))
|
||||
pos += 1
|
||||
self.sections.append(s)
|
||||
self._create_attribute('number-sections', value=len(self.sections))
|
||||
self.add_attribute('number-sections', value=len(self.sections))
|
||||
|
||||
|
||||
class MachOSectionObject(MISPObjectGenerator):
|
||||
class MachOSectionObject(AbstractMISPObjectGenerator):
|
||||
|
||||
def __init__(self, section):
|
||||
# Python3 way
|
||||
|
|
@ -76,13 +76,13 @@ class MachOSectionObject(MISPObjectGenerator):
|
|||
self.generate_attributes()
|
||||
|
||||
def generate_attributes(self):
|
||||
self._create_attribute('name', value=self.section.name)
|
||||
size = self._create_attribute('size-in-bytes', value=self.section.size)
|
||||
self.add_attribute('name', value=self.section.name)
|
||||
size = self.add_attribute('size-in-bytes', value=self.section.size)
|
||||
if int(size.value) > 0:
|
||||
self._create_attribute('entropy', value=self.section.entropy)
|
||||
self._create_attribute('md5', value=md5(self.data).hexdigest())
|
||||
self._create_attribute('sha1', value=sha1(self.data).hexdigest())
|
||||
self._create_attribute('sha256', value=sha256(self.data).hexdigest())
|
||||
self._create_attribute('sha512', value=sha512(self.data).hexdigest())
|
||||
self.add_attribute('entropy', value=self.section.entropy)
|
||||
self.add_attribute('md5', value=md5(self.data).hexdigest())
|
||||
self.add_attribute('sha1', value=sha1(self.data).hexdigest())
|
||||
self.add_attribute('sha256', value=sha256(self.data).hexdigest())
|
||||
self.add_attribute('sha512', value=sha512(self.data).hexdigest())
|
||||
if HAS_PYDEEP:
|
||||
self._create_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode())
|
||||
self.add_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode())
|
||||
|
|
|
|||
|
|
@ -1,142 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from collections import Counter
|
||||
from pymisp import MISPEvent, MISPAttribute, AbstractMISP
|
||||
import os
|
||||
import json
|
||||
import uuid
|
||||
import abc
|
||||
import sys
|
||||
import six # Remove that import when discarding python2 support.
|
||||
|
||||
|
||||
class MISPObjectException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidMISPObject(MISPObjectException):
|
||||
"""Exception raised when an object doesn't contains the required field(s)"""
|
||||
pass
|
||||
|
||||
|
||||
if six.PY2:
|
||||
import warnings
|
||||
warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5")
|
||||
|
||||
|
||||
class MISPObjectReference(AbstractMISP):
|
||||
|
||||
attributes = ['source_uuid', 'destination_uuid', 'relationship_type', 'comment']
|
||||
|
||||
def __init__(self, source_uuid, destination_uuid, relationship_type, comment=None):
|
||||
self.source_uuid = source_uuid
|
||||
self.destination_uuid = destination_uuid
|
||||
self.relationship_type = relationship_type
|
||||
self.comment = comment
|
||||
|
||||
|
||||
class MISPObjectAttribute(AbstractMISP):
|
||||
|
||||
# This list is very limited and hardcoded to fit the current needs (file/pe/pesection creation): MISPAttriute will follow the
|
||||
# same spec and just add one attribute: object_relation
|
||||
attributes = ['object_relation', 'value', 'type', 'category', 'disable_correlation', 'to_ids',
|
||||
'data', 'encrypt', 'distribution', 'comment']
|
||||
|
||||
def __init__(self, definition, object_relation, value, **kwargs):
|
||||
self.object_relation = object_relation
|
||||
self.value = value
|
||||
# Initialize the new MISPAttribute
|
||||
# Get the misp attribute type from the definition
|
||||
self.type = kwargs.pop('type', None)
|
||||
if self.type is None:
|
||||
self.type = definition['misp-attribute']
|
||||
self.disable_correlation = kwargs.pop('disable_correlation', None)
|
||||
if self.disable_correlation is None:
|
||||
# The correlation can be disabled by default in the object definition.
|
||||
# Use this value if it isn't overloaded by the object
|
||||
self.disable_correlation = definition.get('disable_correlation')
|
||||
self.to_ids = kwargs.pop('to_ids', None)
|
||||
if self.to_ids is None:
|
||||
# Same for the to_ids flag
|
||||
self.to_ids = definition.get('to_ids')
|
||||
# Initialise rest of the values
|
||||
for k, v in kwargs.items():
|
||||
self[k] = v
|
||||
# FIXME: dirty hack until all the classes are ported to the new format but we get the default values
|
||||
temp_attribute = MISPAttribute()
|
||||
temp_attribute.set_all_values(**self)
|
||||
# Update default values
|
||||
self.from_dict(**temp_attribute.to_dict())
|
||||
|
||||
|
||||
class MISPObjectGenerator(AbstractMISP):
|
||||
|
||||
attributes = ['name', 'meta-category', 'uuid', 'description', 'version', 'Attribute']
|
||||
|
||||
def __init__(self, template_dir):
|
||||
"""This class is used to fill a new MISP object with the default values defined in the object template
|
||||
* template is the path to the template within the misp-object repository
|
||||
* misp_objects_path is the path to the misp-object repository
|
||||
"""
|
||||
self.misp_objects_path = os.path.join(
|
||||
os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)),
|
||||
'data', 'misp-objects', 'objects')
|
||||
with open(os.path.join(self.misp_objects_path, template_dir, 'definition.json'), 'r') as f:
|
||||
self.definition = json.load(f)
|
||||
self.misp_event = MISPEvent()
|
||||
self.name = self.definition['name']
|
||||
setattr(self, 'meta-category', self.definition['meta-category'])
|
||||
self.template_uuid = self.definition['uuid']
|
||||
self.description = self.definition['description']
|
||||
self.version = self.definition['version']
|
||||
self.uuid = str(uuid.uuid4())
|
||||
self.Attribute = []
|
||||
self.references = []
|
||||
|
||||
def _create_attribute(self, object_type, **value):
|
||||
if value.get('value') is None:
|
||||
return None
|
||||
attribute = MISPObjectAttribute(self.definition['attributes'][object_type], object_type, **value)
|
||||
self.Attribute.append(attribute)
|
||||
return attribute
|
||||
|
||||
def to_dict(self, strict=True):
|
||||
if strict:
|
||||
self._validate()
|
||||
return super(MISPObjectGenerator, self).to_dict()
|
||||
|
||||
def to_json(self, strict=True):
|
||||
if strict:
|
||||
self._validate()
|
||||
return super(MISPObjectGenerator, self).to_json()
|
||||
|
||||
def _validate(self):
|
||||
"""Make sure the object we're creating has the required fields"""
|
||||
all_object_relations = []
|
||||
for a in self.Attribute:
|
||||
all_object_relations.append(a.object_relation)
|
||||
count_relations = dict(Counter(all_object_relations))
|
||||
for key, counter in count_relations.items():
|
||||
if counter == 1:
|
||||
continue
|
||||
if not self.definition['attributes'][key].get('multiple'):
|
||||
raise InvalidMISPObject('Multiple occurrences of {} is not allowed'.format(key))
|
||||
all_attribute_names = set(count_relations.keys())
|
||||
if self.definition.get('requiredOneOf'):
|
||||
if not set(self.definition['requiredOneOf']) & all_attribute_names:
|
||||
raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self.definition['requiredOneOf'])))
|
||||
if self.definition.get('required'):
|
||||
for r in self.definition.get('required'):
|
||||
if r not in all_attribute_names:
|
||||
raise InvalidMISPObject('{} is required'.format(r))
|
||||
return True
|
||||
|
||||
def add_reference(self, destination_uuid, relationship_type, comment=None):
|
||||
"""Add a link (uuid) to an other object"""
|
||||
self.references.append(MISPObjectReference(self.uuid, destination_uuid, relationship_type, comment))
|
||||
|
||||
@abc.abstractmethod
|
||||
def generate_attributes(self):
|
||||
"""Contains the logic where all the values of the object are gathered"""
|
||||
pass
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from pymisp.tools import MISPObjectGenerator
|
||||
from pymisp.defaultobjects import AbstractMISPObjectGenerator
|
||||
from io import BytesIO
|
||||
from hashlib import md5, sha1, sha256, sha512
|
||||
from datetime import datetime
|
||||
|
|
@ -21,7 +21,7 @@ except ImportError:
|
|||
HAS_PYDEEP = False
|
||||
|
||||
|
||||
class PEObject(MISPObjectGenerator):
|
||||
class PEObject(AbstractMISPObjectGenerator):
|
||||
|
||||
def __init__(self, parsed=None, filepath=None, pseudofile=None):
|
||||
if not HAS_PYDEEP:
|
||||
|
|
@ -74,10 +74,10 @@ class PEObject(MISPObjectGenerator):
|
|||
return 'unknown'
|
||||
|
||||
def generate_attributes(self):
|
||||
self._create_attribute('type', value=self._get_pe_type())
|
||||
self.add_attribute('type', value=self._get_pe_type())
|
||||
# General information
|
||||
self._create_attribute('entrypoint-address', value=self.pe.entrypoint)
|
||||
self._create_attribute('compilation-timestamp', value=datetime.utcfromtimestamp(self.pe.header.time_date_stamps).isoformat())
|
||||
self.add_attribute('entrypoint-address', value=self.pe.entrypoint)
|
||||
self.add_attribute('compilation-timestamp', value=datetime.utcfromtimestamp(self.pe.header.time_date_stamps).isoformat())
|
||||
# self.imphash = self.pe.get_imphash()
|
||||
try:
|
||||
if (self.pe.has_resources and
|
||||
|
|
@ -85,15 +85,15 @@ class PEObject(MISPObjectGenerator):
|
|||
self.pe.resources_manager.version.has_string_file_info and
|
||||
self.pe.resources_manager.version.string_file_info.langcode_items):
|
||||
fileinfo = dict(self.pe.resources_manager.version.string_file_info.langcode_items[0].items.items())
|
||||
self._create_attribute('original-filename', value=fileinfo.get('OriginalFilename'))
|
||||
self._create_attribute('internal-filename', value=fileinfo.get('InternalName'))
|
||||
self._create_attribute('file-description', value=fileinfo.get('FileDescription'))
|
||||
self._create_attribute('file-version', value=fileinfo.get('FileVersion'))
|
||||
self._create_attribute('lang-id', value=self.pe.resources_manager.version.string_file_info.langcode_items[0].key)
|
||||
self._create_attribute('product-name', value=fileinfo.get('ProductName'))
|
||||
self._create_attribute('product-version', value=fileinfo.get('ProductVersion'))
|
||||
self._create_attribute('company-name', value=fileinfo.get('CompanyName'))
|
||||
self._create_attribute('legal-copyright', value=fileinfo.get('LegalCopyright'))
|
||||
self.add_attribute('original-filename', value=fileinfo.get('OriginalFilename'))
|
||||
self.add_attribute('internal-filename', value=fileinfo.get('InternalName'))
|
||||
self.add_attribute('file-description', value=fileinfo.get('FileDescription'))
|
||||
self.add_attribute('file-version', value=fileinfo.get('FileVersion'))
|
||||
self.add_attribute('lang-id', value=self.pe.resources_manager.version.string_file_info.langcode_items[0].key)
|
||||
self.add_attribute('product-name', value=fileinfo.get('ProductName'))
|
||||
self.add_attribute('product-version', value=fileinfo.get('ProductVersion'))
|
||||
self.add_attribute('company-name', value=fileinfo.get('CompanyName'))
|
||||
self.add_attribute('legal-copyright', value=fileinfo.get('LegalCopyright'))
|
||||
except lief.read_out_of_bound:
|
||||
# The file is corrupted
|
||||
pass
|
||||
|
|
@ -106,14 +106,14 @@ class PEObject(MISPObjectGenerator):
|
|||
self.add_reference(s.uuid, 'included-in', 'Section {} of PE'.format(pos))
|
||||
if ((self.pe.entrypoint >= section.virtual_address) and
|
||||
(self.pe.entrypoint < (section.virtual_address + section.virtual_size))):
|
||||
self._create_attribute('entrypoint-section|position', value='{}|{}'.format(section.name, pos))
|
||||
self.add_attribute('entrypoint-section|position', value='{}|{}'.format(section.name, pos))
|
||||
pos += 1
|
||||
self.sections.append(s)
|
||||
self._create_attribute('number-sections', value=len(self.sections))
|
||||
self.add_attribute('number-sections', value=len(self.sections))
|
||||
# TODO: TLSSection / DIRECTORY_ENTRY_TLS
|
||||
|
||||
|
||||
class PESectionObject(MISPObjectGenerator):
|
||||
class PESectionObject(AbstractMISPObjectGenerator):
|
||||
|
||||
def __init__(self, section):
|
||||
# Python3 way
|
||||
|
|
@ -124,13 +124,13 @@ class PESectionObject(MISPObjectGenerator):
|
|||
self.generate_attributes()
|
||||
|
||||
def generate_attributes(self):
|
||||
self._create_attribute('name', value=self.section.name)
|
||||
size = self._create_attribute('size-in-bytes', value=self.section.size)
|
||||
self.add_attribute('name', value=self.section.name)
|
||||
size = self.add_attribute('size-in-bytes', value=self.section.size)
|
||||
if int(size.value) > 0:
|
||||
self._create_attribute('entropy', value=self.section.entropy)
|
||||
self._create_attribute('md5', value=md5(self.data).hexdigest())
|
||||
self._create_attribute('sha1', value=sha1(self.data).hexdigest())
|
||||
self._create_attribute('sha256', value=sha256(self.data).hexdigest())
|
||||
self._create_attribute('sha512', value=sha512(self.data).hexdigest())
|
||||
self.add_attribute('entropy', value=self.section.entropy)
|
||||
self.add_attribute('md5', value=md5(self.data).hexdigest())
|
||||
self.add_attribute('sha1', value=sha1(self.data).hexdigest())
|
||||
self.add_attribute('sha256', value=sha256(self.data).hexdigest())
|
||||
self.add_attribute('sha512', value=sha512(self.data).hexdigest())
|
||||
if HAS_PYDEEP:
|
||||
self._create_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode())
|
||||
self.add_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode())
|
||||
|
|
|
|||
|
|
@ -230,18 +230,18 @@ class TestOffline(unittest.TestCase):
|
|||
if seos:
|
||||
for s in seos:
|
||||
to_return['objects'].append(s)
|
||||
if s.references:
|
||||
to_return['references'] += s.references
|
||||
if s.ObjectReference:
|
||||
to_return['references'] += s.ObjectReference
|
||||
|
||||
if peo:
|
||||
to_return['objects'].append(peo)
|
||||
if peo.references:
|
||||
to_return['references'] += peo.references
|
||||
if peo.ObjectReference:
|
||||
to_return['references'] += peo.ObjectReference
|
||||
|
||||
if fo:
|
||||
to_return['objects'].append(fo)
|
||||
if fo.references:
|
||||
to_return['references'] += fo.references
|
||||
if fo.ObjectReference:
|
||||
to_return['references'] += fo.ObjectReference
|
||||
return json.dumps(to_return, cls=MISPEncode)
|
||||
|
||||
def test_objects(self, m):
|
||||
|
|
|
|||
Loading…
Reference in New Issue