From cf257493f7cf377a734d91477498b6175dd50f26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 26 Sep 2016 00:26:09 +0200 Subject: [PATCH] First batch of changes, will be squashed --- pymisp/__init__.py | 3 +- pymisp/api.py | 147 +++++++----------------------- pymisp/exceptions.py | 31 +++++++ pymisp/mispevent.py | 211 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 279 insertions(+), 113 deletions(-) create mode 100644 pymisp/exceptions.py create mode 100644 pymisp/mispevent.py diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 36eb6d2..70200cc 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,3 +1,4 @@ __version__ = '2.4.51.1' -from .api import PyMISP, PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey +from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey +from .api import PyMISP diff --git a/pymisp/api.py b/pymisp/api.py index c44577b..b20deaf 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -15,8 +15,6 @@ except ImportError: from urlparse import urljoin from io import BytesIO import zipfile -import warnings -import functools try: import requests @@ -25,6 +23,8 @@ except ImportError: HAVE_REQUESTS = False from . import __version__ +from .exceptions import PyMISPError, NewEventError, NewAttributeError, SearchError, MissingDependency, NoURL, NoKey +from .mispevent import MISPEvent, MISPAttribute # Least dirty way to support python 2 and 3 try: @@ -56,36 +56,6 @@ class analysis(object): completed = 2 -class PyMISPError(Exception): - def __init__(self, message): - super(PyMISPError, self).__init__(message) - self.message = message - - -class NewEventError(PyMISPError): - pass - - -class NewAttributeError(PyMISPError): - pass - - -class SearchError(PyMISPError): - pass - - -class MissingDependency(PyMISPError): - pass - - -class NoURL(PyMISPError): - pass - - -class NoKey(PyMISPError): - pass - - class PyMISP(object): """ Python API for MISP @@ -102,6 +72,9 @@ class PyMISP(object): :param cert: Client certificate, as described there: http://docs.python-requests.org/en/master/user/advanced/#ssl-cert-verification """ +# TODO: interactive script to create a MISP event from scratch +# TODO: a parser to verify the validity of an event + # So it can may be accessed from the misp object. distributions = distributions threat_level = threat_level @@ -138,6 +111,11 @@ class PyMISP(object): self.categories = self.describe_types['result']['categories'] self.types = self.describe_types['result']['types'] self.category_type_mapping = self.describe_types['result']['category_type_mappings'] + # New in 2.5.52 + if self.describe_types['result'].get('sane_defaults'): + self.sane_default = self.describe_types['result']['sane_defaults'] + else: + self.sane_default = {} def __prepare_session(self, output='json'): """ @@ -311,53 +289,16 @@ class PyMISP(object): # ############################################## def _prepare_full_event(self, distribution, threat_level_id, analysis, info, date=None, published=False): - to_return = {'Event': {}} - # Setup details of a new event - if distribution not in [0, 1, 2, 3]: - raise NewEventError('{} is invalid, the distribution has to be in 0, 1, 2, 3'.format(distribution)) - if threat_level_id not in [1, 2, 3, 4]: - raise NewEventError('{} is invalid, the threat_level_id has to be in 1, 2, 3, 4'.format(threat_level_id)) - if analysis not in [0, 1, 2]: - raise NewEventError('{} is invalid, the analysis has to be in 0, 1, 2'.format(analysis)) - if date is None: - date = datetime.date.today().isoformat() - if published not in [True, False]: - raise NewEventError('{} is invalid, published has to be True or False'.format(published)) - to_return['Event'] = {'distribution': distribution, 'info': info, 'date': date, 'published': published, - 'threat_level_id': threat_level_id, 'analysis': analysis} - return to_return + misp_event = MISPEvent(self.describe_types['result']) + misp_event.set_values(info, distribution, threat_level_id, analysis, date) + if published: + misp_event.publish() + return misp_event.dump() - def _prepare_full_attribute(self, category, type_value, value, to_ids, comment=None, distribution=None): - to_return = {} - if category not in self.categories: - raise NewAttributeError('{} is invalid, category has to be in {}'.format(category, (', '.join(self.categories)))) - - if type_value not in self.types: - raise NewAttributeError('{} is invalid, type_value has to be in {}'.format(type_value, (', '.join(self.types)))) - - if type_value not in self.category_type_mapping[category]: - raise NewAttributeError('{} and {} is an invalid combinaison, type_value for this category has to be in {}'.format(type_value, category, (', '.join(self.category_type_mapping[category])))) - to_return['type'] = type_value - to_return['category'] = category - - if to_ids not in [True, False]: - raise NewAttributeError('{} is invalid, to_ids has to be True or False'.format(to_ids)) - to_return['to_ids'] = to_ids - - if distribution is not None: - distribution = int(distribution) - # If None: take the default value of the event - if distribution not in [None, 0, 1, 2, 3, 5]: - raise NewAttributeError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 5 or None'.format(distribution)) - if distribution is not None: - to_return['distribution'] = distribution - - to_return['value'] = value - - if comment is not None: - to_return['comment'] = comment - - return to_return + def _prepare_full_attribute(self, category, type_value, value, to_ids, comment=None, distribution=5): + misp_attribute = MISPAttribute(self.categories, self.types, self.category_type_mapping) + misp_attribute.set_values(type_value, value, category, to_ids, comment, distribution) + return misp_attribute.dump() def _prepare_update(self, event): # Cleanup the received event to make it publishable @@ -381,30 +322,30 @@ class PyMISP(object): # ########## Helpers ########## def get(self, eid): - response = self.get_event(int(eid)) - return response + return self.get_event(eid) def get_stix(self, **kwargs): - response = self.get_stix_event(**kwargs) - return response + return self.get_stix_event(**kwargs) def update(self, event): eid = event['Event']['id'] - response = self.update_event(eid, event) - return response - - def new_event(self, distribution=None, threat_level_id=None, analysis=None, info=None, date=None, published=False): - data = self._prepare_full_event(distribution, threat_level_id, analysis, info, date, published) - response = self.add_event(data) - return response + return self.update_event(eid, event) def publish(self, event): if event['Event']['published']: return {'error': 'Already published'} event = self._prepare_update(event) event['Event']['published'] = True - response = self.update_event(event['Event']['id'], event) - return response + return self.update_event(event['Event']['id'], event) + + def change_threat_level(self, event, threat_level_id): + event['Event']['threat_level_id'] = threat_level_id + self._prepare_update(event) + return self.update_event(event['Event']['id'], event) + + def new_event(self, distribution=None, threat_level_id=None, analysis=None, info=None, date=None, published=False): + data = self._prepare_full_event(distribution, threat_level_id, analysis, info, date, published) + return self.add_event(data) def add_tag(self, event, tag): session = self.__prepare_session() @@ -418,12 +359,6 @@ class PyMISP(object): response = session.post(urljoin(self.root_url, 'events/removeTag'), data=json.dumps(to_post)) return self._check_response(response) - def change_threat_level(self, event, threat_level_id): - event['Event']['threat_level_id'] = threat_level_id - self._prepare_update(event) - response = self.update_event(event['Event']['id'], event) - return response - # ##### File attributes ##### def _send_attributes(self, event, attributes, proposal=False): @@ -708,21 +643,9 @@ class PyMISP(object): # ######### Upload samples through the API ######### # ################################################## - def _create_event(self, distribution, threat_level_id, analysis, info): - # Setup details of a new event - if distribution not in [0, 1, 2, 3]: - raise NewEventError('{} is invalid, the distribution has to be in 0, 1, 2, 3'.format(distribution)) - if threat_level_id not in [1, 2, 3, 4]: - raise NewEventError('{} is invalid, the threat_level_id has to be in 1, 2, 3, 4'.format(threat_level_id)) - if analysis not in [0, 1, 2]: - raise NewEventError('{} is invalid, the analysis has to be in 0, 1, 2'.format(analysis)) - return {'distribution': int(distribution), 'info': info, - 'threat_level_id': int(threat_level_id), 'analysis': analysis} - def prepare_attribute(self, event_id, distribution, to_ids, category, comment, info, analysis, threat_level_id): to_post = {'request': {}} - authorized_categs = ['Payload delivery', 'Artifacts dropped', 'Payload installation', 'External analysis', 'Network activity', 'Antivirus detection'] if event_id is not None: try: @@ -731,7 +654,7 @@ class PyMISP(object): pass if not isinstance(event_id, int): # New event - to_post['request'] = self._create_event(distribution, threat_level_id, analysis, info) + to_post['request'] = self._prepare_full_event(distribution, threat_level_id, analysis, info) else: to_post['request']['event_id'] = int(event_id) @@ -739,8 +662,8 @@ class PyMISP(object): raise NewAttributeError('{} is invalid, to_ids has to be True or False'.format(to_ids)) to_post['request']['to_ids'] = to_ids - if category not in authorized_categs: - raise NewAttributeError('{} is invalid, category has to be in {}'.format(category, (', '.join(authorized_categs)))) + if category not in self.categories: + raise NewAttributeError('{} is invalid, category has to be in {}'.format(category, (', '.join(self.categories)))) to_post['request']['category'] = category to_post['request']['comment'] = comment diff --git a/pymisp/exceptions.py b/pymisp/exceptions.py new file mode 100644 index 0000000..f4db340 --- /dev/null +++ b/pymisp/exceptions.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +class PyMISPError(Exception): + def __init__(self, message): + super(PyMISPError, self).__init__(message) + self.message = message + + +class NewEventError(PyMISPError): + pass + + +class NewAttributeError(PyMISPError): + pass + + +class SearchError(PyMISPError): + pass + + +class MissingDependency(PyMISPError): + pass + + +class NoURL(PyMISPError): + pass + + +class NoKey(PyMISPError): + pass diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py new file mode 100644 index 0000000..2d181b4 --- /dev/null +++ b/pymisp/mispevent.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import datetime +import time +import json + +from .exceptions import PyMISPError, NewEventError, NewAttributeError + + +class MISPAttribute(object): + + def __init__(self, categories, types, category_type_mapping): + self.categories = categories + self.types = types + self.category_type_mapping = category_type_mapping + self.new = True + + # Default values + self.category = None + self.type = None + self.value = None + self.to_ids = False + self.comment = '' + self.distribution = 5 + + def set_values(self, type_value, value, category, to_ids, comment, distribution): + self._validate(type_value, value, category, to_ids, comment, distribution) + self.type = type_value + self.value = value + self.category = category + self.to_ids = to_ids + self.comment = comment + self.distribution = distribution + + def set_values_existing_attribute(self, attribute_id, uuid, timestamp, sharing_group_id, deleted, SharingGroup, ShadowAttribute): + self.new = False + self.id = int(attribute_id) + self.uuid = uuid + self.timestamp = datetime.datetime.fromtimestamp(timestamp) + self.sharing_group_id = int(sharing_group_id) + self.deleted = deleted + self.SharingGroup = SharingGroup + self.ShadowAttribute = ShadowAttribute + + def _validate(self, type_value, value, category, to_ids, comment, distribution): + if category not in self.categories: + raise NewAttributeError('{} is invalid, category has to be in {}'.format(category, (', '.join(self.categories)))) + if type_value not in self.types: + raise NewAttributeError('{} is invalid, type_value has to be in {}'.format(type_value, (', '.join(self.types)))) + if type_value not in self.category_type_mapping[category]: + raise NewAttributeError('{} and {} is an invalid combinaison, type_value for this category has to be in {}'.capitalizeformat(type_value, category, (', '.join(self.category_type_mapping[category])))) + if to_ids not in [True, False]: + raise NewAttributeError('{} is invalid, to_ids has to be True or False'.format(to_ids)) + if distribution not in [0, 1, 2, 3, 5]: + raise NewAttributeError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 5'.format(distribution)) + + def dump(self): + to_return = {'type': self.type, 'category': self.category, 'to_ids': self.to_ids, + 'distribution': self.distribution, 'value': self.value, + 'comment': self.comment} + if not self.new: + to_return.update( + {'id': self.id, 'uuid': self.uuid, + 'timestamp': int(time.mktime(self.timestamp.timetuple())), + 'sharing_group_id': self.sharing_group_id, 'deleted': self.deleted, + 'SharingGroup': self.SharingGroup, 'ShadowAttribute': self.ShadowAttribute}) + return to_return + + +class MISPEvent(object): + + def __init__(self, describe_types): + self.categories = describe_types['categories'] + self.types = describe_types['types'] + self.category_type_mapping = describe_types['category_type_mappings'] + self.sane_default = describe_types['sane_defaults'] + self.new = True + self.dump_full = False + + # Default values + self.distribution = 3 + self.threat_level_id = 2 + self.analysis = 0 + self.info = '' + self.published = False + self.date = datetime.date.today() + self.attributes = [] + + def _validate(self, distribution, threat_level_id, analysis): + if distribution not in [0, 1, 2, 3]: + raise NewEventError('{} is invalid, the distribution has to be in 0, 1, 2, 3'.format(distribution)) + if threat_level_id not in [1, 2, 3, 4]: + raise NewEventError('{} is invalid, the threat_level has to be in 1, 2, 3, 4'.format(threat_level_id)) + if analysis not in [0, 1, 2]: + raise NewEventError('{} is invalid, the analysis has to be in 0, 1, 2'.format(analysis)) + + def load(self, json_event): + self.new = False + self.dump_full = True + loaded = json.loads(json_event) + if loaded.get('response'): + e = loaded.get('response')[0].get('Event') + else: + e = loaded.get('Event') + if not e: + raise PyMISPError('Invalid event') + try: + date = datetime.date(*map(int, e['date'].split('-'))) + except: + raise NewEventError('{} is an invalid date.'.format(e['date'])) + self.set_values(e['info'], int(e['distribution']), int(e['threat_level_id']), int(e['analysis']), date) + if e['published']: + self.publish() + self.set_values_existing_event( + e['id'], e['orgc_id'], e['org_id'], e['uuid'], + e['attribute_count'], e['proposal_email_lock'], e['locked'], + e['publish_timestamp'], e['sharing_group_id'], e['Org'], e['Orgc'], + e['ShadowAttribute'], e['RelatedEvent']) + self.attributes = [] + for a in e['Attribute']: + attribute = MISPAttribute(self.categories, self.types, self.category_type_mapping) + attribute.set_values(a['type'], a['value'], a['category'], a['to_ids'], + a['comment'], int(a['distribution'])) + attribute.set_values_existing_attribute(a['id'], a['uuid'], a['timestamp'], + a['sharing_group_id'], a['deleted'], + a['SharingGroup'], a['ShadowAttribute']) + self.attributes.append(attribute) + + def dump(self): + to_return = {'Event': {}} + to_return['Event'] = {'distribution': self.distribution, 'info': self.info, + 'date': self.date.isoformat(), 'published': self.published, + 'threat_level_id': self.threat_level_id, + 'analysis': self.analysis, 'Attribute': []} + if not self.new: + to_return['Event'].update( + {'id': self.id, 'orgc_id': self.orgc_id, 'org_id': self.org_id, + 'uuid': self.uuid, 'sharing_group_id': self.sharing_group_id}) + if self.dump_full: + to_return['Event'].update( + {'locked': self.locked, 'attribute_count': self.attribute_count, + 'RelatedEvent': self.RelatedEvent, 'Orgc': self.Orgc, + 'ShadowAttribute': self.ShadowAttribute, 'Org': self.Org, + 'proposal_email_lock': self.proposal_email_lock, + 'publish_timestamp': int(time.mktime(self.publish_timestamp.timetuple()))}) + to_return['Event']['Attribute'] = [a.dump() for a in self.attributes] + return json.dumps(to_return) + + def set_values(self, info, distribution=3, threat_level_id=2, analysis=0, date=None): + self._validate(distribution, threat_level_id, analysis) + self.info = info + self.distribution = distribution + self.threat_level_id = threat_level_id + self.analysis = analysis + if not date: + self.date = datetime.date.today() + else: + self.date = date + + def set_values_existing_event(self, event_id, orgc_id, org_id, uuid, attribute_count, + proposal_email_lock, locked, publish_timestamp, + sharing_group_id, Org, Orgc, ShadowAttribute, + RelatedEvent): + self.id = int(event_id) + self.orgc_id = int(orgc_id) + self.org_id = int(org_id) + self.uuid = uuid + self.attribute_count = int(attribute_count) + self.proposal_email_lock = proposal_email_lock + self.locked = locked + self.publish_timestamp = datetime.datetime.fromtimestamp(publish_timestamp) + self.sharing_group_id = int(sharing_group_id) + self.Org = Org + self.Orgc = Orgc + self.ShadowAttribute = ShadowAttribute + self.RelatedEvent = RelatedEvent + + def publish(self): + self.publish = True + + def unpublish(self): + self.publish = False + + def prepare_for_update(self): + self.unpublish() + self.dump_full = False + + def add_attribute(self, type_value, value, **kwargs): + if not self.sane_default.get(type_value): + raise NewAttributeError("{} is an invalid type. Can only be one of the following: {}".format(type_value, ', '.join(self.types))) + defaults = self.sane_default[type_value] + if kwargs.get('category'): + category = kwargs.get('category') + else: + category = defaults['default_category'] + if kwargs.get('to_ids'): + to_ids = bool(int(kwargs.get('to_ids'))) + else: + to_ids = bool(int(defaults['to_ids'])) + if kwargs.get('comment'): + comment = kwargs.get('comment') + else: + comment = None + if kwargs.get('distribution'): + distribution = int(kwargs.get('distribution')) + else: + distribution = 5 + attribute = MISPAttribute(self.categories, self.types, self.category_type_mapping) + attribute.set_values(type_value, value, category, to_ids, comment, distribution) + self.attributes.append(attribute)