MISP
/
PyMISP
mirror of https://github.com/MISP/PyMISP
2
0
Fork 0
Code Issues Releases Wiki Activity

Some more refactoring and cleanup

pull/111/head
Raphaël Vinot 2017-08-30 12:47:32 +02:00
parent 69ede74ce7
commit 74037cb6fa
10 changed files with 200 additions and 212 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
pymisp/__init__.py
View File

@ -1,11 +1,10 @@
__version__ = '2.4.77'
__version__ = '2.4.80'
try:
from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate # noqa
from .api import PyMISP # noqa
from .abstract import AbstractMISP, MISPEncode # noqa
from .defaultobjects import MISPObject, AbstractMISPObjectGenerator # noqa
from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull # noqa
from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull, MISPObjectReference, MISPObjectAttribute, MISPObject, AbstractMISPObjectGenerator # noqa
from .tools import Neo4j # noqa
from .tools import stix # noqa
except ImportError:

195
pymisp/defaultobjects.py
View File

@ -1,195 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import abc
import os
import json
import sys
import uuid
from collections import Counter
from .abstract import AbstractMISP
from .exceptions import UnknownMISPObjectTemplate, InvalidMISPObject
import six # Remove that import when discarding python2 support.
if six.PY2:
import warnings
warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5")
class MISPObjectReference(AbstractMISP):
attributes = ['source_uuid', 'destination_uuid', 'relationship_type', 'comment', 'uuid', 'deleted']
def __init__(self):
super(MISPObjectReference, self).__init__()
def from_dict(self, source_uuid, destination_uuid, relationship_type, comment=None, **kwargs):
self.source_uuid = source_uuid
self.destination_uuid = destination_uuid
self.relationship_type = relationship_type
self.comment = comment
for k, v in kwargs:
setattr(self, k, v)
class MISPObjectAttribute(AbstractMISP):
# This list is very limited and hardcoded to fit the current needs (file/pe/pesection creation): MISPAttriute will follow the
# same spec and just add one attribute: object_relation
attributes = ['object_relation', 'value', 'type', 'category', 'disable_correlation', 'to_ids',
'data', 'encrypt', 'distribution', 'comment', 'uuid', 'event_id']
def __init__(self, definition):
super(MISPObjectAttribute, self).__init__()
self.definition = definition
def from_dict(self, object_relation, value, **kwargs):
from .mispevent import MISPAttribute
self.object_relation = object_relation
self.value = value
# Initialize the new MISPAttribute
# Get the misp attribute type from the definition
self.type = kwargs.pop('type', None)
if self.type is None:
self.type = self.definition.get('misp-attribute')
self.disable_correlation = kwargs.pop('disable_correlation', None)
if self.disable_correlation is None:
# The correlation can be disabled by default in the object definition.
# Use this value if it isn't overloaded by the object
self.disable_correlation = self.definition.get('disable_correlation')
self.to_ids = kwargs.pop('to_ids', None)
if self.to_ids is None:
# Same for the to_ids flag
self.to_ids = self.definition.get('to_ids')
# FIXME: dirty hack until all the classes are ported to the new format but we get the default values
# Initialise rest of the values
for k, v in kwargs.items():
setattr(self, k, v)
temp_attribute = MISPAttribute()
temp_attribute.set_all_values(**self)
# Update default values
for k, v in temp_attribute.to_dict().items():
setattr(self, k, v)
class MISPObject(AbstractMISP):
attributes = ['name', 'meta-category', 'uuid', 'description', 'template_version', 'template_uuid', 'Attribute']
def __init__(self, name, strict=True):
super(MISPObject, self).__init__()
self.strict = strict
self.name = name
self.misp_objects_path = os.path.join(
os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)),
'data', 'misp-objects', 'objects')
if os.path.exists(os.path.join(self.misp_objects_path, self.name, 'definition.json')):
self.known_template = True
else:
if self.strict:
raise UnknownMISPObjectTemplate('{} is unknown in the MISP object directory.')
else:
self.known_template = False
if self.known_template:
with open(os.path.join(self.misp_objects_path, self.name, 'definition.json'), 'r') as f:
self.definition = json.load(f)
setattr(self, 'meta-category', self.definition['meta-category'])
self.template_uuid = self.definition['uuid']
self.description = self.definition['description']
self.template_version = self.definition['version']
else:
# FIXME We need to set something for meta-category, template_uuid, description and template_version
pass
self.uuid = str(uuid.uuid4())
self.Attribute = []
self.ObjectReference = []
def from_dict(self, **kwargs):
if self.known_template:
if kwargs.get('template_uuid') and kwargs['template_uuid'] != self.template_uuid:
if self.strict:
raise UnknownMISPObjectTemplate('UUID of the object is different from the one of the template.')
else:
self.known_template = False
if kwargs.get('template_version') and int(kwargs['template_version']) != self.template_version:
if self.strict:
raise UnknownMISPObjectTemplate('Version of the object ({}) is different from the one of the template ({}).'.format(kwargs['template_version'], self.template_version))
else:
self.known_template = False
for key, value in kwargs.items():
if key == 'Attribute':
for v in value:
self.add_attribute(**v)
elif key == 'ObjectReference':
for v in value:
self.add_reference(**v)
else:
setattr(self, key, value)
def to_dict(self, strict=True):
if strict or self.strict and self.known_template:
self._validate()
return super(MISPObject, self).to_dict()
def to_json(self, strict=True):
if strict or self.strict and self.known_template:
self._validate()
return super(MISPObject, self).to_json()
def _validate(self):
"""Make sure the object we're creating has the required fields"""
all_object_relations = []
for a in self.Attribute:
all_object_relations.append(a.object_relation)
count_relations = dict(Counter(all_object_relations))
for key, counter in count_relations.items():
if counter == 1:
continue
if not self.definition['attributes'][key].get('multiple'):
raise InvalidMISPObject('Multiple occurrences of {} is not allowed'.format(key))
all_attribute_names = set(count_relations.keys())
if self.definition.get('requiredOneOf'):
if not set(self.definition['requiredOneOf']) & all_attribute_names:
raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self.definition['requiredOneOf'])))
if self.definition.get('required'):
for r in self.definition.get('required'):
if r not in all_attribute_names:
raise InvalidMISPObject('{} is required'.format(r))
return True
def add_reference(self, destination_uuid, relationship_type, comment=None, **kwargs):
"""Add a link (uuid) to an other object"""
if kwargs.get('source_uuid'):
# Load existing object
source_uuid = kwargs.get('source_uuid')
else:
# New reference
source_uuid = self.uuid
reference = MISPObjectReference()
reference.from_dict(source_uuid=source_uuid, destination_uuid=destination_uuid,
relationship_type=relationship_type, comment=comment, **kwargs)
self.ObjectReference.append(reference)
def add_attribute(self, object_relation, **value):
if value.get('value') is None:
return None
if self.known_template:
attribute = MISPObjectAttribute(self.definition['attributes'][object_relation])
else:
attribute = MISPObjectAttribute({})
attribute.from_dict(object_relation, **value)
self.Attribute.append(attribute)
return attribute
@six.add_metaclass(abc.ABCMeta) # Remove that line when discarding python2 support.
# Python3 way: class MISPObjectGenerator(metaclass=abc.ABCMeta):
class AbstractMISPObjectGenerator(MISPObject):
@abc.abstractmethod
def generate_attributes(self):
"""Contains the logic where all the values of the object are gathered"""
pass

194
pymisp/mispevent.py
View File

@ -6,11 +6,24 @@ import time
import json
from json import JSONEncoder
import os
import warnings
import base64
from io import BytesIO
from zipfile import ZipFile
import hashlib
import abc
import sys
import uuid
from collections import Counter
from .abstract import AbstractMISP
from .exceptions import UnknownMISPObjectTemplate, InvalidMISPObject, PyMISPError, NewEventError, NewAttributeError
import six # Remove that import when discarding python2 support.
if six.PY2:
import warnings
warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5")
try:
from dateutil.parser import parse
@ -36,14 +49,10 @@ except ImportError:
except ImportError:
has_pyme = False
from .exceptions import PyMISPError, NewEventError, NewAttributeError
from .defaultobjects import MISPObject
# Least dirty way to support python 2 and 3
try:
basestring
unicode
warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5")
except NameError:
basestring = str
unicode = str
@ -124,6 +133,10 @@ class MISPAttribute(object):
return {self.uuid: False}
def set_all_values(self, **kwargs):
# to be deprecated
self.from_dict(**kwargs)
def from_dict(self, **kwargs):
if kwargs.get('type') and kwargs.get('category'):
if kwargs['type'] not in self.category_type_mapping[kwargs['category']]:
raise NewAttributeError('{} and {} is an invalid combination, type for this category has to be in {}'.format(kwargs.get('type'), kwargs.get('category'), (', '.join(self.category_type_mapping[kwargs['category']]))))
@ -622,3 +635,174 @@ class MISPEvent(object):
else:
attribute.set_all_values(type=type, value=value, **kwargs)
self.attributes.append(attribute)
class MISPObjectReference(AbstractMISP):
attributes = ['source_uuid', 'destination_uuid', 'relationship_type', 'comment', 'uuid', 'deleted']
def __init__(self):
super(MISPObjectReference, self).__init__()
def from_dict(self, source_uuid, destination_uuid, relationship_type, comment=None, **kwargs):
self.source_uuid = source_uuid
self.destination_uuid = destination_uuid
self.relationship_type = relationship_type
self.comment = comment
for k, v in kwargs:
setattr(self, k, v)
class MISPObjectAttribute(MISPAttribute, AbstractMISP):
# This list is very limited and hardcoded to fit the current needs (file/pe/pesection creation): MISPAttriute will follow the
# same spec and just add one attribute: object_relation
attributes = ['object_relation', 'value', 'type', 'category', 'disable_correlation', 'to_ids',
'data', 'encrypt', 'distribution', 'comment', 'uuid', 'event_id']
def __init__(self, definition):
MISPAttribute.__init__(self)
AbstractMISP.__init__(self)
self.definition = definition
def from_dict(self, object_relation, value, **kwargs):
self.object_relation = object_relation
self.value = value
# Initialize the new MISPAttribute
# Get the misp attribute type from the definition
self.type = kwargs.pop('type', None)
if self.type is None:
self.type = self.definition.get('misp-attribute')
self.disable_correlation = kwargs.pop('disable_correlation', None)
if self.disable_correlation is None:
# The correlation can be disabled by default in the object definition.
# Use this value if it isn't overloaded by the object
self.disable_correlation = self.definition.get('disable_correlation')
self.to_ids = kwargs.pop('to_ids', None)
if self.to_ids is None:
# Same for the to_ids flag
self.to_ids = self.definition.get('to_ids')
# FIXME: dirty hack until all the classes are ported to the new format but we get the default values
kwargs.update(**self)
MISPAttribute.from_dict(self, **kwargs)
class MISPObject(AbstractMISP):
attributes = ['name', 'meta-category', 'uuid', 'description', 'template_version', 'template_uuid', 'Attribute']
def __init__(self, name, strict=True):
super(MISPObject, self).__init__()
self.strict = strict
self.name = name
self.misp_objects_path = os.path.join(
os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)),
'data', 'misp-objects', 'objects')
if os.path.exists(os.path.join(self.misp_objects_path, self.name, 'definition.json')):
self.known_template = True
else:
if self.strict:
raise UnknownMISPObjectTemplate('{} is unknown in the MISP object directory.')
else:
self.known_template = False
if self.known_template:
with open(os.path.join(self.misp_objects_path, self.name, 'definition.json'), 'r') as f:
self.definition = json.load(f)
setattr(self, 'meta-category', self.definition['meta-category'])
self.template_uuid = self.definition['uuid']
self.description = self.definition['description']
self.template_version = self.definition['version']
else:
# FIXME We need to set something for meta-category, template_uuid, description and template_version
pass
self.uuid = str(uuid.uuid4())
self.Attribute = []
self.ObjectReference = []
def from_dict(self, **kwargs):
if self.known_template:
if kwargs.get('template_uuid') and kwargs['template_uuid'] != self.template_uuid:
if self.strict:
raise UnknownMISPObjectTemplate('UUID of the object is different from the one of the template.')
else:
self.known_template = False
if kwargs.get('template_version') and int(kwargs['template_version']) != self.template_version:
if self.strict:
raise UnknownMISPObjectTemplate('Version of the object ({}) is different from the one of the template ({}).'.format(kwargs['template_version'], self.template_version))
else:
self.known_template = False
for key, value in kwargs.items():
if key == 'Attribute':
for v in value:
self.add_attribute(**v)
elif key == 'ObjectReference':
for v in value:
self.add_reference(**v)
else:
setattr(self, key, value)
def to_dict(self, strict=True):
if strict or self.strict and self.known_template:
self._validate()
return super(MISPObject, self).to_dict()
def to_json(self, strict=True):
if strict or self.strict and self.known_template:
self._validate()
return super(MISPObject, self).to_json()
def _validate(self):
"""Make sure the object we're creating has the required fields"""
all_object_relations = []
for a in self.Attribute:
all_object_relations.append(a.object_relation)
count_relations = dict(Counter(all_object_relations))
for key, counter in count_relations.items():
if counter == 1:
continue
if not self.definition['attributes'][key].get('multiple'):
raise InvalidMISPObject('Multiple occurrences of {} is not allowed'.format(key))
all_attribute_names = set(count_relations.keys())
if self.definition.get('requiredOneOf'):
if not set(self.definition['requiredOneOf']) & all_attribute_names:
raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self.definition['requiredOneOf'])))
if self.definition.get('required'):
for r in self.definition.get('required'):
if r not in all_attribute_names:
raise InvalidMISPObject('{} is required'.format(r))
return True
def add_reference(self, destination_uuid, relationship_type, comment=None, **kwargs):
"""Add a link (uuid) to an other object"""
if kwargs.get('source_uuid'):
# Load existing object
source_uuid = kwargs.get('source_uuid')
else:
# New reference
source_uuid = self.uuid
reference = MISPObjectReference()
reference.from_dict(source_uuid=source_uuid, destination_uuid=destination_uuid,
relationship_type=relationship_type, comment=comment, **kwargs)
self.ObjectReference.append(reference)
def add_attribute(self, object_relation, **value):
if value.get('value') is None:
return None
if self.known_template:
attribute = MISPObjectAttribute(self.definition['attributes'][object_relation])
else:
attribute = MISPObjectAttribute({})
attribute.from_dict(object_relation, **value)
self.Attribute.append(attribute)
return attribute
@six.add_metaclass(abc.ABCMeta) # Remove that line when discarding python2 support.
# Python3 way: class MISPObjectGenerator(metaclass=abc.ABCMeta):
class AbstractMISPObjectGenerator(MISPObject):
@abc.abstractmethod
def generate_attributes(self):
"""Contains the logic where all the values of the object are gathered"""
pass

4
pymisp/tools/create_misp_object.py
View File

@ -1,8 +1,8 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from pymisp.tools import FileObject, PEObject, ELFObject, MachOObject
from pymisp.exceptions import MISPObjectException
from . import FileObject, PEObject, ELFObject, MachOObject
from ..exceptions import MISPObjectException
import warnings
try:

2
pymisp/tools/elfobject.py
View File

@ -1,7 +1,7 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from pymisp.defaultobjects import AbstractMISPObjectGenerator
from .. import AbstractMISPObjectGenerator
from io import BytesIO
from hashlib import md5, sha1, sha256, sha512
import warnings

2
pymisp/tools/fileobject.py
View File

@ -1,7 +1,7 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from pymisp.defaultobjects import AbstractMISPObjectGenerator
from .. import AbstractMISPObjectGenerator
import os
from io import BytesIO
from hashlib import md5, sha1, sha256, sha512

2
pymisp/tools/machoobject.py
View File

@ -1,7 +1,7 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from pymisp.defaultobjects import AbstractMISPObjectGenerator
from .. import AbstractMISPObjectGenerator
from io import BytesIO
from hashlib import md5, sha1, sha256, sha512
import warnings

4
pymisp/tools/neo4j.py
View File

@ -3,7 +3,7 @@
import glob
import os
from pymisp import MISPEvent
from .. import MISPEvent
try:
from py2neo import authenticate, Graph, Node, Relationship
@ -54,5 +54,5 @@ class Neo4j():
av = Relationship(attr_node, "is", val)
s = val | ev | av
tx.merge(s)
#tx.graph.push(s)
# tx.graph.push(s)
tx.commit()

2
pymisp/tools/openioc.py
View File

@ -3,7 +3,7 @@
import os
from pymisp import MISPEvent
from .. import MISPEvent
try:
from bs4 import BeautifulSoup
has_bs4 = True

2
pymisp/tools/peobject.py
View File

@ -1,7 +1,7 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from pymisp.defaultobjects import AbstractMISPObjectGenerator
from .. import AbstractMISPObjectGenerator
from io import BytesIO
from hashlib import md5, sha1, sha256, sha512
from datetime import datetime