mirror of https://github.com/MISP/PyMISP
Refactor all the things
Add script for MISP core, make everything generic.pull/111/head
Raphaël Vinot
parent
bd6deba55e
commit
c09ce0032c
|
|
@ -7,7 +7,6 @@ import traceback
|
|||
from keys import misp_url, misp_key, misp_verifycert
|
||||
import glob
|
||||
import argparse
|
||||
import json
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description='Extract indicators out of binaries and add MISP objects to a MISP instance.')
|
||||
|
|
@ -25,20 +24,17 @@ if __name__ == '__main__':
|
|||
|
||||
if seos:
|
||||
for s in seos:
|
||||
obj, refs = s
|
||||
template_id = pymisp.get_object_template_id(obj['name'])
|
||||
r = pymisp.add_object(args.event, template_id, obj)
|
||||
template_id = pymisp.get_object_template_id(s['name'])
|
||||
r = pymisp.add_object(args.event, template_id, s)
|
||||
|
||||
if peo:
|
||||
obj, refs = peo
|
||||
template_id = pymisp.get_object_template_id(obj['name'])
|
||||
r = pymisp.add_object(args.event, template_id, obj)
|
||||
for ref in refs:
|
||||
r = pymisp.add_object_reference(obj['uuid'], ref)
|
||||
template_id = pymisp.get_object_template_id(peo['name'])
|
||||
r = pymisp.add_object(args.event, template_id, peo)
|
||||
for ref in peo.references:
|
||||
r = pymisp.add_object_reference(ref)
|
||||
|
||||
if fo:
|
||||
obj, refs = fo
|
||||
template_id = pymisp.get_object_template_id(obj['name'])
|
||||
response = pymisp.add_object(args.event, template_id, obj)
|
||||
for ref in refs:
|
||||
r = pymisp.add_object_reference(obj['uuid'], ref)
|
||||
template_id = pymisp.get_object_template_id(fo['name'])
|
||||
response = pymisp.add_object(args.event, template_id, fo)
|
||||
for ref in fo.references:
|
||||
r = pymisp.add_object_reference(ref)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,63 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from pymisp import MISPEncode
|
||||
from pymisp.tools import make_binary_objects
|
||||
import argparse
|
||||
import json
|
||||
|
||||
|
||||
def check():
|
||||
missing_dependencies = {'pydeep': False, 'lief': False, 'magic': False, 'pymisp': False}
|
||||
try:
|
||||
import pymisp # noqa
|
||||
except ImportError:
|
||||
missing_dependencies['pymisp'] = 'Please install pydeep: pip install pymisp'
|
||||
try:
|
||||
import pydeep # noqa
|
||||
except ImportError:
|
||||
missing_dependencies['pydeep'] = 'Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git'
|
||||
try:
|
||||
import lief # noqa
|
||||
except ImportError:
|
||||
missing_dependencies['lief'] = 'Please install lief, documentation here: https://github.com/lief-project/LIEF'
|
||||
try:
|
||||
import magic # noqa
|
||||
except ImportError:
|
||||
missing_dependencies['magic'] = 'Please install python-magic: pip install python-magic.'
|
||||
return json.dumps(missing_dependencies)
|
||||
|
||||
|
||||
def make_objects(path):
|
||||
to_return = {'objects': [], 'references': []}
|
||||
fo, peo, seos = make_binary_objects(path)
|
||||
|
||||
if seos:
|
||||
for s in seos:
|
||||
to_return['objects'].append(s)
|
||||
if s.references:
|
||||
to_return['references'] += s.references
|
||||
|
||||
if peo:
|
||||
to_return['objects'].append(peo)
|
||||
if peo.references:
|
||||
to_return['references'] += peo.references
|
||||
|
||||
if fo:
|
||||
to_return['objects'].append(fo)
|
||||
if fo.references:
|
||||
to_return['references'] += fo.references
|
||||
return json.dumps(to_return, cls=MISPEncode)
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description='Extract indicators out of binaries and returns MISP objects.')
|
||||
group = parser.add_mutually_exclusive_group()
|
||||
group.add_argument("-p", "--path", help="Path to process.")
|
||||
group.add_argument("-c", "--check", action='store_true', help="Check the dependencies.")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.check:
|
||||
print(check())
|
||||
if args.path:
|
||||
obj = make_objects(args.path)
|
||||
print(obj)
|
||||
|
|
@ -2,7 +2,7 @@ __version__ = '2.4.77'
|
|||
|
||||
from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey
|
||||
from .api import PyMISP
|
||||
from .abstract import AbstractMISP
|
||||
from .abstract import AbstractMISP, MISPEncode
|
||||
from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull
|
||||
from .tools import Neo4j
|
||||
from .tools import stix
|
||||
|
|
|
|||
|
|
@ -4,9 +4,18 @@
|
|||
import six # Remove that import when discarding python2 support.
|
||||
import abc
|
||||
import json
|
||||
from json import JSONEncoder
|
||||
import collections
|
||||
|
||||
|
||||
class MISPEncode(JSONEncoder):
|
||||
|
||||
def default(self, obj):
|
||||
if isinstance(obj, AbstractMISP):
|
||||
return obj.jsonable()
|
||||
return JSONEncoder.default(self, obj)
|
||||
|
||||
|
||||
@six.add_metaclass(abc.ABCMeta) # Remove that line when discarding python2 support.
|
||||
# Python3 way: class MISPObjectGenerator(metaclass=abc.ABCMeta):
|
||||
class AbstractMISP(collections.MutableMapping):
|
||||
|
|
@ -48,8 +57,11 @@ class AbstractMISP(collections.MutableMapping):
|
|||
to_return[attribute] = val
|
||||
return to_return
|
||||
|
||||
def jsonable(self):
|
||||
return self.to_dict()
|
||||
|
||||
def to_json(self):
|
||||
return json.dumps(self.to_dict(), indent=4, sort_keys=True)
|
||||
return json.dumps(self.to_dict(), cls=MISPEncode)
|
||||
|
||||
def __getitem__(self, key):
|
||||
if self.__check_dict_key(key):
|
||||
|
|
|
|||
|
|
@ -1585,13 +1585,13 @@ class PyMISP(object):
|
|||
def add_object(self, event_id, template_id, misp_object):
|
||||
session = self.__prepare_session()
|
||||
url = urljoin(self.root_url, 'objects/add/{}/{}'.format(event_id, template_id))
|
||||
response = session.post(url, data=json.dumps(misp_object))
|
||||
response = session.post(url, data=misp_object.to_json())
|
||||
return self._check_response(response)
|
||||
|
||||
def add_object_reference(self, parent_uuid, misp_object_reference):
|
||||
def add_object_reference(self, misp_object_reference):
|
||||
session = self.__prepare_session()
|
||||
url = urljoin(self.root_url, 'object_references/add/{}'.format(parent_uuid))
|
||||
response = session.post(url, data=json.dumps(misp_object_reference))
|
||||
url = urljoin(self.root_url, 'object_references/add')
|
||||
response = session.post(url, data=misp_object_reference.to_json())
|
||||
return self._check_response(response)
|
||||
|
||||
def get_object_templates_list(self):
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from pymisp.tools import FileObject, PEObject, MISPObjectException
|
||||
|
|
@ -17,11 +17,11 @@ class FileTypeNotImplemented(MISPObjectException):
|
|||
def make_pe_objects(lief_parsed, misp_file):
|
||||
misp_pe = PEObject(parsed=lief_parsed)
|
||||
misp_file.add_reference(misp_pe.uuid, 'included-in', 'PE indicators')
|
||||
file_object = misp_file.dump()
|
||||
pe_object = misp_pe.dump()
|
||||
file_object = misp_file
|
||||
pe_object = misp_pe
|
||||
pe_sections = []
|
||||
for s in misp_pe.sections:
|
||||
pe_sections.append(s.dump())
|
||||
pe_sections.append(s)
|
||||
return file_object, pe_object, pe_sections
|
||||
|
||||
|
||||
|
|
@ -45,5 +45,5 @@ def make_binary_objects(filepath):
|
|||
print('\tParser error: ', e)
|
||||
except FileTypeNotImplemented as e:
|
||||
print(e)
|
||||
file_object = misp_file.dump()
|
||||
file_object = misp_file.to_json()
|
||||
return file_object, None, None
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from io import BytesIO
|
|||
from hashlib import md5, sha1, sha256, sha512
|
||||
import math
|
||||
from collections import Counter
|
||||
import warnings
|
||||
|
||||
try:
|
||||
import pydeep
|
||||
|
|
@ -25,9 +26,9 @@ class FileObject(MISPObjectGenerator):
|
|||
|
||||
def __init__(self, filepath=None, pseudofile=None, filename=None):
|
||||
if not HAS_PYDEEP:
|
||||
raise ImportError("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git")
|
||||
warnings.warn("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git")
|
||||
if not HAS_MAGIC:
|
||||
raise ImportError("Please install python-magic: pip install python-magic.")
|
||||
warnings.warn("Please install python-magic: pip install python-magic.")
|
||||
if filepath:
|
||||
self.filepath = filepath
|
||||
self.filename = os.path.basename(self.filepath)
|
||||
|
|
@ -46,16 +47,18 @@ class FileObject(MISPObjectGenerator):
|
|||
|
||||
def generate_attributes(self):
|
||||
self._create_attribute('filename', value=self.filename)
|
||||
self._create_attribute('size-in-bytes', value=len(self.data))
|
||||
if getattr(self, 'size-in-bytes').value > 0:
|
||||
size = self._create_attribute('size-in-bytes', value=len(self.data))
|
||||
if int(size.value) > 0:
|
||||
self._create_attribute('entropy', value=self.__entropy_H(self.data))
|
||||
self._create_attribute('md5', value=md5(self.data).hexdigest())
|
||||
self._create_attribute('sha1', value=sha1(self.data).hexdigest())
|
||||
self._create_attribute('sha256', value=sha256(self.data).hexdigest())
|
||||
self._create_attribute('sha512', value=sha512(self.data).hexdigest())
|
||||
self._create_attribute('mimetype', value=magic.from_buffer(self.data))
|
||||
self._create_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode())
|
||||
self._create_attribute('malware-sample', value=self.filename.value, data=self.pseudofile)
|
||||
self._create_attribute('malware-sample', value=self.filename, data=self.pseudofile)
|
||||
if HAS_MAGIC:
|
||||
self._create_attribute('mimetype', value=magic.from_buffer(self.data))
|
||||
if HAS_PYDEEP:
|
||||
self._create_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode())
|
||||
|
||||
def __entropy_H(self, data):
|
||||
"""Calculate the entropy of a chunk of data."""
|
||||
|
|
|
|||
|
|
@ -26,16 +26,53 @@ if six.PY2:
|
|||
|
||||
class MISPObjectReference(AbstractMISP):
|
||||
|
||||
attributes = ['uuid', 'relationship_type', 'comment']
|
||||
attributes = ['source_uuid', 'destination_uuid', 'relationship_type', 'comment']
|
||||
|
||||
def __init__(self, uuid, relationship_type, comment=None):
|
||||
self['uuid'] = uuid
|
||||
self['relationship_type'] = relationship_type
|
||||
self['comment'] = comment
|
||||
def __init__(self, source_uuid, destination_uuid, relationship_type, comment=None):
|
||||
self.source_uuid = source_uuid
|
||||
self.destination_uuid = destination_uuid
|
||||
self.relationship_type = relationship_type
|
||||
self.comment = comment
|
||||
|
||||
|
||||
class MISPObjectAttribute(AbstractMISP):
|
||||
|
||||
# This list is very limited and hardcoded to fit the current needs (file/pe/pesection creation): MISPAttriute will follow the
|
||||
# same spec and just add one attribute: object_relation
|
||||
attributes = ['object_relation', 'value', 'type', 'category', 'disable_correlation', 'to_ids',
|
||||
'data', 'encrypt', 'distribution', 'comment']
|
||||
|
||||
def __init__(self, definition, object_relation, value, **kwargs):
|
||||
self.object_relation = object_relation
|
||||
self.value = value
|
||||
# Initialize the new MISPAttribute
|
||||
# Get the misp attribute type from the definition
|
||||
self.type = kwargs.pop('type', None)
|
||||
if self.type is None:
|
||||
self.type = definition['misp-attribute']
|
||||
self.disable_correlation = kwargs.pop('disable_correlation', None)
|
||||
if self.disable_correlation is None:
|
||||
# The correlation can be disabled by default in the object definition.
|
||||
# Use this value if it isn't overloaded by the object
|
||||
self.disable_correlation = definition.get('disable_correlation')
|
||||
self.to_ids = kwargs.pop('to_ids', None)
|
||||
if self.to_ids is None:
|
||||
# Same for the to_ids flag
|
||||
self.to_ids = definition.get('to_ids')
|
||||
# Initialise rest of the values
|
||||
for k, v in kwargs.items():
|
||||
self[k] = v
|
||||
# FIXME: dirty hack until all the classes are ported to the new format but we get the default values
|
||||
temp_attribute = MISPAttribute()
|
||||
temp_attribute.set_all_values(**self)
|
||||
# Update default values
|
||||
self.from_dict(**temp_attribute.to_dict())
|
||||
|
||||
|
||||
class MISPObjectGenerator(AbstractMISP):
|
||||
|
||||
attributes = ['name', 'meta-category', 'uuid', 'description', 'version', 'Attribute']
|
||||
|
||||
def __init__(self, template_dir):
|
||||
"""This class is used to fill a new MISP object with the default values defined in the object template
|
||||
* template is the path to the template within the misp-object repository
|
||||
|
|
@ -46,53 +83,38 @@ class MISPObjectGenerator(AbstractMISP):
|
|||
'data', 'misp-objects', 'objects')
|
||||
with open(os.path.join(self.misp_objects_path, template_dir, 'definition.json'), 'r') as f:
|
||||
self.definition = json.load(f)
|
||||
self.attributes = self.definition['attributes'].keys()
|
||||
self.object_attributes = self.definition['attributes'].keys()
|
||||
self.misp_event = MISPEvent()
|
||||
self.name = self.definition['name']
|
||||
setattr(self, 'meta-category', self.definition['meta-category'])
|
||||
self.uuid = str(uuid.uuid4())
|
||||
self.description = self.definition['description']
|
||||
self.version = self.definition['version']
|
||||
self.Attribute = []
|
||||
self.references = []
|
||||
|
||||
def _create_attribute(self, object_type, **value):
|
||||
if value.get('value') is None:
|
||||
return None
|
||||
# Initialize the new MISPAttribute
|
||||
# Get the misp attribute type from the definition
|
||||
value['type'] = self.definition['attributes'][object_type]['misp-attribute']
|
||||
if value.get('disable_correlation') is None:
|
||||
# The correlation can be disabled by default in the object definition.
|
||||
# Use this value if it isn't overloaded by the object
|
||||
value['disable_correlation'] = self.definition['attributes'][object_type].get('disable_correlation')
|
||||
if value.get('to_ids') is None:
|
||||
# Same for the to_ids flag
|
||||
value['to_ids'] = self.definition['attributes'][object_type].get('to_ids')
|
||||
# Set all the values in the MISP attribute
|
||||
attribute = MISPAttribute(self.misp_event.describe_types)
|
||||
attribute.set_all_values(**value)
|
||||
self[object_type] = attribute
|
||||
attribute = MISPObjectAttribute(self.definition['attributes'][object_type], object_type, **value)
|
||||
self.Attribute.append(attribute)
|
||||
return attribute
|
||||
|
||||
def dump(self, strict=True):
|
||||
"""Create a new object with the values gathered by the sub-class, use the default values from the template if needed"""
|
||||
def to_dict(self, strict=True):
|
||||
if strict:
|
||||
self._validate()
|
||||
# Create an empty object based om the object definition
|
||||
new_object = self.__new_empty_object(self.definition)
|
||||
for object_type, attribute in self.items():
|
||||
# Add all the values as MISPAttributes to the current object
|
||||
if attribute.value is None:
|
||||
continue
|
||||
# Finalize the actual MISP Object
|
||||
# FIXME: This only works on python >= 3.5
|
||||
# new_object['Attribute'].append({'object_relation': object_type, **attribute._json()})
|
||||
# ### BEGIN ####
|
||||
# Because we still need to support old python.
|
||||
temp_attribute = {'object_relation': object_type}
|
||||
temp_attribute.update(attribute._json())
|
||||
new_object['Attribute'].append(temp_attribute)
|
||||
# ### END ####
|
||||
return new_object, [r.to_dict() for r in self.references]
|
||||
return super(MISPObjectGenerator, self).to_dict()
|
||||
|
||||
def to_json(self, strict=True):
|
||||
if strict:
|
||||
self._validate()
|
||||
return super(MISPObjectGenerator, self).to_json()
|
||||
|
||||
def _validate(self):
|
||||
"""Make sure the object we're creating has the required fields"""
|
||||
all_attribute_names = set(self.keys())
|
||||
all_attribute_names = set()
|
||||
for a in self.Attribute:
|
||||
all_attribute_names.add(a.object_relation)
|
||||
if self.definition.get('requiredOneOf'):
|
||||
if not set(self.definition['requiredOneOf']) & all_attribute_names:
|
||||
raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self.definition['requiredOneOf'])))
|
||||
|
|
@ -102,15 +124,9 @@ class MISPObjectGenerator(AbstractMISP):
|
|||
raise InvalidMISPObject('{} is required is required'.format(r))
|
||||
return True
|
||||
|
||||
def add_reference(self, uuid, relationship_type, comment=None):
|
||||
def add_reference(self, destination_uuid, relationship_type, comment=None):
|
||||
"""Add a link (uuid) to an other object"""
|
||||
self.references.append(MISPObjectReference(uuid, relationship_type, comment))
|
||||
|
||||
def __new_empty_object(self, object_definiton):
|
||||
"""Create a new empty object out of the template"""
|
||||
return {'name': object_definiton['name'], 'meta-category': object_definiton['meta-category'],
|
||||
'uuid': self.uuid, 'description': object_definiton['description'],
|
||||
'version': object_definiton['version'], 'Attribute': []}
|
||||
self.references.append(MISPObjectReference(self.uuid, destination_uuid, relationship_type, comment))
|
||||
|
||||
@abc.abstractmethod
|
||||
def generate_attributes(self):
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ from pymisp.tools import MISPObjectGenerator
|
|||
from io import BytesIO
|
||||
from hashlib import md5, sha1, sha256, sha512
|
||||
from datetime import datetime
|
||||
import warnings
|
||||
|
||||
|
||||
try:
|
||||
|
|
@ -24,7 +25,7 @@ class PEObject(MISPObjectGenerator):
|
|||
|
||||
def __init__(self, parsed=None, filepath=None, pseudofile=None):
|
||||
if not HAS_PYDEEP:
|
||||
raise ImportError("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git")
|
||||
warnings.warn("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git")
|
||||
if not HAS_LIEF:
|
||||
raise ImportError('Please install lief, documentation here: https://github.com/lief-project/LIEF')
|
||||
if pseudofile:
|
||||
|
|
@ -120,8 +121,8 @@ class PESectionObject(MISPObjectGenerator):
|
|||
|
||||
def generate_attributes(self):
|
||||
self._create_attribute('name', value=self.section.name)
|
||||
self._create_attribute('size-in-bytes', value=self.section.size)
|
||||
if getattr(self, 'size-in-bytes').value > 0:
|
||||
size = self._create_attribute('size-in-bytes', value=self.section.size)
|
||||
if int(size.value) > 0:
|
||||
self._create_attribute('entropy', value=self.section.entropy)
|
||||
self._create_attribute('md5', value=md5(self.data).hexdigest())
|
||||
self._create_attribute('sha1', value=sha1(self.data).hexdigest())
|
||||
|
|
|
|||
Loading…
Reference in New Issue