mirror of https://github.com/MISP/PyMISP
Some more refactoring and cleanup
parent
69ede74ce7
commit
74037cb6fa
|
@ -1,11 +1,10 @@
|
|||
__version__ = '2.4.77'
|
||||
__version__ = '2.4.80'
|
||||
|
||||
try:
|
||||
from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate # noqa
|
||||
from .api import PyMISP # noqa
|
||||
from .abstract import AbstractMISP, MISPEncode # noqa
|
||||
from .defaultobjects import MISPObject, AbstractMISPObjectGenerator # noqa
|
||||
from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull # noqa
|
||||
from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull, MISPObjectReference, MISPObjectAttribute, MISPObject, AbstractMISPObjectGenerator # noqa
|
||||
from .tools import Neo4j # noqa
|
||||
from .tools import stix # noqa
|
||||
except ImportError:
|
||||
|
|
|
@ -1,195 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import abc
|
||||
import os
|
||||
import json
|
||||
import sys
|
||||
import uuid
|
||||
from collections import Counter
|
||||
|
||||
from .abstract import AbstractMISP
|
||||
from .exceptions import UnknownMISPObjectTemplate, InvalidMISPObject
|
||||
import six # Remove that import when discarding python2 support.
|
||||
|
||||
|
||||
if six.PY2:
|
||||
import warnings
|
||||
warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5")
|
||||
|
||||
|
||||
class MISPObjectReference(AbstractMISP):
|
||||
|
||||
attributes = ['source_uuid', 'destination_uuid', 'relationship_type', 'comment', 'uuid', 'deleted']
|
||||
|
||||
def __init__(self):
|
||||
super(MISPObjectReference, self).__init__()
|
||||
|
||||
def from_dict(self, source_uuid, destination_uuid, relationship_type, comment=None, **kwargs):
|
||||
self.source_uuid = source_uuid
|
||||
self.destination_uuid = destination_uuid
|
||||
self.relationship_type = relationship_type
|
||||
self.comment = comment
|
||||
for k, v in kwargs:
|
||||
setattr(self, k, v)
|
||||
|
||||
|
||||
class MISPObjectAttribute(AbstractMISP):
|
||||
|
||||
# This list is very limited and hardcoded to fit the current needs (file/pe/pesection creation): MISPAttriute will follow the
|
||||
# same spec and just add one attribute: object_relation
|
||||
attributes = ['object_relation', 'value', 'type', 'category', 'disable_correlation', 'to_ids',
|
||||
'data', 'encrypt', 'distribution', 'comment', 'uuid', 'event_id']
|
||||
|
||||
def __init__(self, definition):
|
||||
super(MISPObjectAttribute, self).__init__()
|
||||
self.definition = definition
|
||||
|
||||
def from_dict(self, object_relation, value, **kwargs):
|
||||
from .mispevent import MISPAttribute
|
||||
self.object_relation = object_relation
|
||||
self.value = value
|
||||
# Initialize the new MISPAttribute
|
||||
# Get the misp attribute type from the definition
|
||||
self.type = kwargs.pop('type', None)
|
||||
if self.type is None:
|
||||
self.type = self.definition.get('misp-attribute')
|
||||
self.disable_correlation = kwargs.pop('disable_correlation', None)
|
||||
if self.disable_correlation is None:
|
||||
# The correlation can be disabled by default in the object definition.
|
||||
# Use this value if it isn't overloaded by the object
|
||||
self.disable_correlation = self.definition.get('disable_correlation')
|
||||
self.to_ids = kwargs.pop('to_ids', None)
|
||||
if self.to_ids is None:
|
||||
# Same for the to_ids flag
|
||||
self.to_ids = self.definition.get('to_ids')
|
||||
# FIXME: dirty hack until all the classes are ported to the new format but we get the default values
|
||||
# Initialise rest of the values
|
||||
for k, v in kwargs.items():
|
||||
setattr(self, k, v)
|
||||
temp_attribute = MISPAttribute()
|
||||
temp_attribute.set_all_values(**self)
|
||||
# Update default values
|
||||
for k, v in temp_attribute.to_dict().items():
|
||||
setattr(self, k, v)
|
||||
|
||||
|
||||
class MISPObject(AbstractMISP):
|
||||
|
||||
attributes = ['name', 'meta-category', 'uuid', 'description', 'template_version', 'template_uuid', 'Attribute']
|
||||
|
||||
def __init__(self, name, strict=True):
|
||||
super(MISPObject, self).__init__()
|
||||
self.strict = strict
|
||||
self.name = name
|
||||
self.misp_objects_path = os.path.join(
|
||||
os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)),
|
||||
'data', 'misp-objects', 'objects')
|
||||
if os.path.exists(os.path.join(self.misp_objects_path, self.name, 'definition.json')):
|
||||
self.known_template = True
|
||||
else:
|
||||
if self.strict:
|
||||
raise UnknownMISPObjectTemplate('{} is unknown in the MISP object directory.')
|
||||
else:
|
||||
self.known_template = False
|
||||
if self.known_template:
|
||||
with open(os.path.join(self.misp_objects_path, self.name, 'definition.json'), 'r') as f:
|
||||
self.definition = json.load(f)
|
||||
setattr(self, 'meta-category', self.definition['meta-category'])
|
||||
self.template_uuid = self.definition['uuid']
|
||||
self.description = self.definition['description']
|
||||
self.template_version = self.definition['version']
|
||||
else:
|
||||
# FIXME We need to set something for meta-category, template_uuid, description and template_version
|
||||
pass
|
||||
self.uuid = str(uuid.uuid4())
|
||||
self.Attribute = []
|
||||
self.ObjectReference = []
|
||||
|
||||
def from_dict(self, **kwargs):
|
||||
if self.known_template:
|
||||
if kwargs.get('template_uuid') and kwargs['template_uuid'] != self.template_uuid:
|
||||
if self.strict:
|
||||
raise UnknownMISPObjectTemplate('UUID of the object is different from the one of the template.')
|
||||
else:
|
||||
self.known_template = False
|
||||
if kwargs.get('template_version') and int(kwargs['template_version']) != self.template_version:
|
||||
if self.strict:
|
||||
raise UnknownMISPObjectTemplate('Version of the object ({}) is different from the one of the template ({}).'.format(kwargs['template_version'], self.template_version))
|
||||
else:
|
||||
self.known_template = False
|
||||
|
||||
for key, value in kwargs.items():
|
||||
if key == 'Attribute':
|
||||
for v in value:
|
||||
self.add_attribute(**v)
|
||||
elif key == 'ObjectReference':
|
||||
for v in value:
|
||||
self.add_reference(**v)
|
||||
else:
|
||||
setattr(self, key, value)
|
||||
|
||||
def to_dict(self, strict=True):
|
||||
if strict or self.strict and self.known_template:
|
||||
self._validate()
|
||||
return super(MISPObject, self).to_dict()
|
||||
|
||||
def to_json(self, strict=True):
|
||||
if strict or self.strict and self.known_template:
|
||||
self._validate()
|
||||
return super(MISPObject, self).to_json()
|
||||
|
||||
def _validate(self):
|
||||
"""Make sure the object we're creating has the required fields"""
|
||||
all_object_relations = []
|
||||
for a in self.Attribute:
|
||||
all_object_relations.append(a.object_relation)
|
||||
count_relations = dict(Counter(all_object_relations))
|
||||
for key, counter in count_relations.items():
|
||||
if counter == 1:
|
||||
continue
|
||||
if not self.definition['attributes'][key].get('multiple'):
|
||||
raise InvalidMISPObject('Multiple occurrences of {} is not allowed'.format(key))
|
||||
all_attribute_names = set(count_relations.keys())
|
||||
if self.definition.get('requiredOneOf'):
|
||||
if not set(self.definition['requiredOneOf']) & all_attribute_names:
|
||||
raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self.definition['requiredOneOf'])))
|
||||
if self.definition.get('required'):
|
||||
for r in self.definition.get('required'):
|
||||
if r not in all_attribute_names:
|
||||
raise InvalidMISPObject('{} is required'.format(r))
|
||||
return True
|
||||
|
||||
def add_reference(self, destination_uuid, relationship_type, comment=None, **kwargs):
|
||||
"""Add a link (uuid) to an other object"""
|
||||
if kwargs.get('source_uuid'):
|
||||
# Load existing object
|
||||
source_uuid = kwargs.get('source_uuid')
|
||||
else:
|
||||
# New reference
|
||||
source_uuid = self.uuid
|
||||
reference = MISPObjectReference()
|
||||
reference.from_dict(source_uuid=source_uuid, destination_uuid=destination_uuid,
|
||||
relationship_type=relationship_type, comment=comment, **kwargs)
|
||||
self.ObjectReference.append(reference)
|
||||
|
||||
def add_attribute(self, object_relation, **value):
|
||||
if value.get('value') is None:
|
||||
return None
|
||||
if self.known_template:
|
||||
attribute = MISPObjectAttribute(self.definition['attributes'][object_relation])
|
||||
else:
|
||||
attribute = MISPObjectAttribute({})
|
||||
attribute.from_dict(object_relation, **value)
|
||||
self.Attribute.append(attribute)
|
||||
return attribute
|
||||
|
||||
|
||||
@six.add_metaclass(abc.ABCMeta) # Remove that line when discarding python2 support.
|
||||
# Python3 way: class MISPObjectGenerator(metaclass=abc.ABCMeta):
|
||||
class AbstractMISPObjectGenerator(MISPObject):
|
||||
|
||||
@abc.abstractmethod
|
||||
def generate_attributes(self):
|
||||
"""Contains the logic where all the values of the object are gathered"""
|
||||
pass
|
|
@ -6,11 +6,24 @@ import time
|
|||
import json
|
||||
from json import JSONEncoder
|
||||
import os
|
||||
import warnings
|
||||
import base64
|
||||
from io import BytesIO
|
||||
from zipfile import ZipFile
|
||||
import hashlib
|
||||
import abc
|
||||
import sys
|
||||
import uuid
|
||||
from collections import Counter
|
||||
|
||||
from .abstract import AbstractMISP
|
||||
from .exceptions import UnknownMISPObjectTemplate, InvalidMISPObject, PyMISPError, NewEventError, NewAttributeError
|
||||
|
||||
|
||||
import six # Remove that import when discarding python2 support.
|
||||
|
||||
if six.PY2:
|
||||
import warnings
|
||||
warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5")
|
||||
|
||||
try:
|
||||
from dateutil.parser import parse
|
||||
|
@ -36,14 +49,10 @@ except ImportError:
|
|||
except ImportError:
|
||||
has_pyme = False
|
||||
|
||||
from .exceptions import PyMISPError, NewEventError, NewAttributeError
|
||||
from .defaultobjects import MISPObject
|
||||
|
||||
# Least dirty way to support python 2 and 3
|
||||
try:
|
||||
basestring
|
||||
unicode
|
||||
warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5")
|
||||
except NameError:
|
||||
basestring = str
|
||||
unicode = str
|
||||
|
@ -124,6 +133,10 @@ class MISPAttribute(object):
|
|||
return {self.uuid: False}
|
||||
|
||||
def set_all_values(self, **kwargs):
|
||||
# to be deprecated
|
||||
self.from_dict(**kwargs)
|
||||
|
||||
def from_dict(self, **kwargs):
|
||||
if kwargs.get('type') and kwargs.get('category'):
|
||||
if kwargs['type'] not in self.category_type_mapping[kwargs['category']]:
|
||||
raise NewAttributeError('{} and {} is an invalid combination, type for this category has to be in {}'.format(kwargs.get('type'), kwargs.get('category'), (', '.join(self.category_type_mapping[kwargs['category']]))))
|
||||
|
@ -622,3 +635,174 @@ class MISPEvent(object):
|
|||
else:
|
||||
attribute.set_all_values(type=type, value=value, **kwargs)
|
||||
self.attributes.append(attribute)
|
||||
|
||||
|
||||
class MISPObjectReference(AbstractMISP):
|
||||
|
||||
attributes = ['source_uuid', 'destination_uuid', 'relationship_type', 'comment', 'uuid', 'deleted']
|
||||
|
||||
def __init__(self):
|
||||
super(MISPObjectReference, self).__init__()
|
||||
|
||||
def from_dict(self, source_uuid, destination_uuid, relationship_type, comment=None, **kwargs):
|
||||
self.source_uuid = source_uuid
|
||||
self.destination_uuid = destination_uuid
|
||||
self.relationship_type = relationship_type
|
||||
self.comment = comment
|
||||
for k, v in kwargs:
|
||||
setattr(self, k, v)
|
||||
|
||||
|
||||
class MISPObjectAttribute(MISPAttribute, AbstractMISP):
|
||||
|
||||
# This list is very limited and hardcoded to fit the current needs (file/pe/pesection creation): MISPAttriute will follow the
|
||||
# same spec and just add one attribute: object_relation
|
||||
attributes = ['object_relation', 'value', 'type', 'category', 'disable_correlation', 'to_ids',
|
||||
'data', 'encrypt', 'distribution', 'comment', 'uuid', 'event_id']
|
||||
|
||||
def __init__(self, definition):
|
||||
MISPAttribute.__init__(self)
|
||||
AbstractMISP.__init__(self)
|
||||
self.definition = definition
|
||||
|
||||
def from_dict(self, object_relation, value, **kwargs):
|
||||
self.object_relation = object_relation
|
||||
self.value = value
|
||||
# Initialize the new MISPAttribute
|
||||
# Get the misp attribute type from the definition
|
||||
self.type = kwargs.pop('type', None)
|
||||
if self.type is None:
|
||||
self.type = self.definition.get('misp-attribute')
|
||||
self.disable_correlation = kwargs.pop('disable_correlation', None)
|
||||
if self.disable_correlation is None:
|
||||
# The correlation can be disabled by default in the object definition.
|
||||
# Use this value if it isn't overloaded by the object
|
||||
self.disable_correlation = self.definition.get('disable_correlation')
|
||||
self.to_ids = kwargs.pop('to_ids', None)
|
||||
if self.to_ids is None:
|
||||
# Same for the to_ids flag
|
||||
self.to_ids = self.definition.get('to_ids')
|
||||
# FIXME: dirty hack until all the classes are ported to the new format but we get the default values
|
||||
kwargs.update(**self)
|
||||
MISPAttribute.from_dict(self, **kwargs)
|
||||
|
||||
|
||||
class MISPObject(AbstractMISP):
|
||||
|
||||
attributes = ['name', 'meta-category', 'uuid', 'description', 'template_version', 'template_uuid', 'Attribute']
|
||||
|
||||
def __init__(self, name, strict=True):
|
||||
super(MISPObject, self).__init__()
|
||||
self.strict = strict
|
||||
self.name = name
|
||||
self.misp_objects_path = os.path.join(
|
||||
os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)),
|
||||
'data', 'misp-objects', 'objects')
|
||||
if os.path.exists(os.path.join(self.misp_objects_path, self.name, 'definition.json')):
|
||||
self.known_template = True
|
||||
else:
|
||||
if self.strict:
|
||||
raise UnknownMISPObjectTemplate('{} is unknown in the MISP object directory.')
|
||||
else:
|
||||
self.known_template = False
|
||||
if self.known_template:
|
||||
with open(os.path.join(self.misp_objects_path, self.name, 'definition.json'), 'r') as f:
|
||||
self.definition = json.load(f)
|
||||
setattr(self, 'meta-category', self.definition['meta-category'])
|
||||
self.template_uuid = self.definition['uuid']
|
||||
self.description = self.definition['description']
|
||||
self.template_version = self.definition['version']
|
||||
else:
|
||||
# FIXME We need to set something for meta-category, template_uuid, description and template_version
|
||||
pass
|
||||
self.uuid = str(uuid.uuid4())
|
||||
self.Attribute = []
|
||||
self.ObjectReference = []
|
||||
|
||||
def from_dict(self, **kwargs):
|
||||
if self.known_template:
|
||||
if kwargs.get('template_uuid') and kwargs['template_uuid'] != self.template_uuid:
|
||||
if self.strict:
|
||||
raise UnknownMISPObjectTemplate('UUID of the object is different from the one of the template.')
|
||||
else:
|
||||
self.known_template = False
|
||||
if kwargs.get('template_version') and int(kwargs['template_version']) != self.template_version:
|
||||
if self.strict:
|
||||
raise UnknownMISPObjectTemplate('Version of the object ({}) is different from the one of the template ({}).'.format(kwargs['template_version'], self.template_version))
|
||||
else:
|
||||
self.known_template = False
|
||||
|
||||
for key, value in kwargs.items():
|
||||
if key == 'Attribute':
|
||||
for v in value:
|
||||
self.add_attribute(**v)
|
||||
elif key == 'ObjectReference':
|
||||
for v in value:
|
||||
self.add_reference(**v)
|
||||
else:
|
||||
setattr(self, key, value)
|
||||
|
||||
def to_dict(self, strict=True):
|
||||
if strict or self.strict and self.known_template:
|
||||
self._validate()
|
||||
return super(MISPObject, self).to_dict()
|
||||
|
||||
def to_json(self, strict=True):
|
||||
if strict or self.strict and self.known_template:
|
||||
self._validate()
|
||||
return super(MISPObject, self).to_json()
|
||||
|
||||
def _validate(self):
|
||||
"""Make sure the object we're creating has the required fields"""
|
||||
all_object_relations = []
|
||||
for a in self.Attribute:
|
||||
all_object_relations.append(a.object_relation)
|
||||
count_relations = dict(Counter(all_object_relations))
|
||||
for key, counter in count_relations.items():
|
||||
if counter == 1:
|
||||
continue
|
||||
if not self.definition['attributes'][key].get('multiple'):
|
||||
raise InvalidMISPObject('Multiple occurrences of {} is not allowed'.format(key))
|
||||
all_attribute_names = set(count_relations.keys())
|
||||
if self.definition.get('requiredOneOf'):
|
||||
if not set(self.definition['requiredOneOf']) & all_attribute_names:
|
||||
raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self.definition['requiredOneOf'])))
|
||||
if self.definition.get('required'):
|
||||
for r in self.definition.get('required'):
|
||||
if r not in all_attribute_names:
|
||||
raise InvalidMISPObject('{} is required'.format(r))
|
||||
return True
|
||||
|
||||
def add_reference(self, destination_uuid, relationship_type, comment=None, **kwargs):
|
||||
"""Add a link (uuid) to an other object"""
|
||||
if kwargs.get('source_uuid'):
|
||||
# Load existing object
|
||||
source_uuid = kwargs.get('source_uuid')
|
||||
else:
|
||||
# New reference
|
||||
source_uuid = self.uuid
|
||||
reference = MISPObjectReference()
|
||||
reference.from_dict(source_uuid=source_uuid, destination_uuid=destination_uuid,
|
||||
relationship_type=relationship_type, comment=comment, **kwargs)
|
||||
self.ObjectReference.append(reference)
|
||||
|
||||
def add_attribute(self, object_relation, **value):
|
||||
if value.get('value') is None:
|
||||
return None
|
||||
if self.known_template:
|
||||
attribute = MISPObjectAttribute(self.definition['attributes'][object_relation])
|
||||
else:
|
||||
attribute = MISPObjectAttribute({})
|
||||
attribute.from_dict(object_relation, **value)
|
||||
self.Attribute.append(attribute)
|
||||
return attribute
|
||||
|
||||
|
||||
@six.add_metaclass(abc.ABCMeta) # Remove that line when discarding python2 support.
|
||||
# Python3 way: class MISPObjectGenerator(metaclass=abc.ABCMeta):
|
||||
class AbstractMISPObjectGenerator(MISPObject):
|
||||
|
||||
@abc.abstractmethod
|
||||
def generate_attributes(self):
|
||||
"""Contains the logic where all the values of the object are gathered"""
|
||||
pass
|
||||
|
|
|
@ -1,8 +1,8 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from pymisp.tools import FileObject, PEObject, ELFObject, MachOObject
|
||||
from pymisp.exceptions import MISPObjectException
|
||||
from . import FileObject, PEObject, ELFObject, MachOObject
|
||||
from ..exceptions import MISPObjectException
|
||||
import warnings
|
||||
|
||||
try:
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from pymisp.defaultobjects import AbstractMISPObjectGenerator
|
||||
from .. import AbstractMISPObjectGenerator
|
||||
from io import BytesIO
|
||||
from hashlib import md5, sha1, sha256, sha512
|
||||
import warnings
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from pymisp.defaultobjects import AbstractMISPObjectGenerator
|
||||
from .. import AbstractMISPObjectGenerator
|
||||
import os
|
||||
from io import BytesIO
|
||||
from hashlib import md5, sha1, sha256, sha512
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from pymisp.defaultobjects import AbstractMISPObjectGenerator
|
||||
from .. import AbstractMISPObjectGenerator
|
||||
from io import BytesIO
|
||||
from hashlib import md5, sha1, sha256, sha512
|
||||
import warnings
|
||||
|
|
|
@ -3,7 +3,7 @@
|
|||
|
||||
import glob
|
||||
import os
|
||||
from pymisp import MISPEvent
|
||||
from .. import MISPEvent
|
||||
|
||||
try:
|
||||
from py2neo import authenticate, Graph, Node, Relationship
|
||||
|
|
|
@ -3,7 +3,7 @@
|
|||
|
||||
import os
|
||||
|
||||
from pymisp import MISPEvent
|
||||
from .. import MISPEvent
|
||||
try:
|
||||
from bs4 import BeautifulSoup
|
||||
has_bs4 = True
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from pymisp.defaultobjects import AbstractMISPObjectGenerator
|
||||
from .. import AbstractMISPObjectGenerator
|
||||
from io import BytesIO
|
||||
from hashlib import md5, sha1, sha256, sha512
|
||||
from datetime import datetime
|
||||
|
|
Loading…
Reference in New Issue