PyMISP/pymisp/api.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

""" Python API using the REST interface of MISP """

import json
import datetime
import os
import base64
import re

try:
    from urllib.parse import urljoin
except ImportError:
    from urlparse import urljoin
from io import BytesIO
import zipfile
import warnings
import functools

try:
    import requests
    HAVE_REQUESTS = True
except ImportError:
    HAVE_REQUESTS = False

from . import __version__

# Least dirty way to support python 2 and 3
try:
    basestring
except NameError:
    basestring = str


class distributions(object):
    """Enumeration of the available distributions."""
    your_organization = 0
    this_community = 1
    connected_communities = 2
    all_communities = 3


class threat_level(object):
    """Enumeration of the available threat levels."""
    high = 1
    medium = 2
    low = 3
    undefined = 4


class analysis(object):
    """Enumeration of the available analysis statuses."""
    initial = 0
    ongoing = 1
    completed = 2


class PyMISPError(Exception):
    def __init__(self, message):
        super(PyMISPError, self).__init__(message)
        self.message = message


class NewEventError(PyMISPError):
    pass


class NewAttributeError(PyMISPError):
    pass


class SearchError(PyMISPError):
    pass


class MissingDependency(PyMISPError):
    pass


class NoURL(PyMISPError):
    pass


class NoKey(PyMISPError):
    pass


class PyMISP(object):
    """
        Python API for MISP

        :param url: URL of the MISP instance you want to connect to
        :param key: API key of the user you want to use
        :param ssl: can be True or False (to check ot not the validity
                    of the certificate. Or a CA_BUNDLE in case of self
                    signed certiifcate (the concatenation of all the
                    *.crt of the chain)
        :param out_type: Type of object (json) NOTE: XML output isn't supported anymore, keeping the flag for compatibility reasons.
        :param debug: print all the messages received from the server
        :param proxies: Proxy dict as describes here: http://docs.python-requests.org/en/master/user/advanced/#proxies
        :param cert: Client certificate, as described there: http://docs.python-requests.org/en/master/user/advanced/#ssl-cert-verification
    """

    # So it can may be accessed from the misp object.
    distributions = distributions
    threat_level = threat_level
    analysis = analysis

    def __init__(self, url, key, ssl=True, out_type='json', debug=False, proxies=None, cert=None):
        if not url:
            raise NoURL('Please provide the URL of your MISP instance.')
        if not key:
            raise NoKey('Please provide your authorization key.')

        self.root_url = url
        self.key = key
        self.ssl = ssl
        self.proxies = proxies
        self.cert = cert
        if out_type != 'json':
            raise PyMISPError('The only output type supported by PyMISP is JSON. If you still rely on XML, use PyMISP v2.4.49')
        self.debug = debug

        try:
            # Make sure the MISP instance is working and the URL is valid
            self.get_version()
        except Exception as e:
            raise PyMISPError('Unable to connect to MISP ({}). Please make sure the API key and the URL are correct (http/https is required): {}'.format(self.root_url, e))

        session = self.__prepare_session()
        response = session.get(urljoin(self.root_url, 'attributes/describeTypes.json'))
        self.describe_types = self._check_response(response)
        if self.describe_types.get('error'):
            for e in self.describe_types.get('error'):
                raise PyMISPError('Failed: {}'.format(e))

        self.categories = self.describe_types['result']['categories']
        self.types = self.describe_types['result']['types']
        self.category_type_mapping = self.describe_types['result']['category_type_mappings']

    def __prepare_session(self, output='json'):
        """
            Prepare the headers of the session
        """
        if not HAVE_REQUESTS:
            raise MissingDependency('Missing dependency, install requests (`pip install requests`)')
        session = requests.Session()
        session.verify = self.ssl
        session.proxies = self.proxies
        session.cert = self.cert
        session.headers.update(
            {'Authorization': self.key,
             'Accept': 'application/{}'.format(output),
             'content-type': 'application/{}'.format(output)})
        return session

    def flatten_error_messages(self, response):
        messages = []
        if response.get('error'):
            if isinstance(response['error'], list):
                for e in response['errors']:
                    messages.append(e['error']['value'][0])
            else:
                messages.append(['error'])
        elif response.get('errors'):
            if isinstance(response['errors'], dict):
                for where, errors in response['errors'].items():
                    for e in errors:
                        for type_e, msgs in e.items():
                            for m in msgs:
                                messages.append('Error in {}: {}'.format(where, m))
        return messages

    def _check_response(self, response):
        if response.status_code >= 500:
            response.raise_for_status()
        try:
            to_return = response.json()
        except:
            if self.debug:
                print(response.text)
            raise PyMISPError('Unknown error: {}'.format(response.text))

        errors = []
        if isinstance(to_return, list):
            to_return = {'response': to_return}
        if to_return.get('error'):
            if not isinstance(to_return['error'], list):
                errors.append(to_return['error'])
            else:
                errors += to_return['error']

        if 400 <= response.status_code < 500:
            if to_return.get('error') is None and to_return.get('message'):
                errors.append(to_return['message'])
            else:
                errors.append(basestring(response.status_code))
        errors += self.flatten_error_messages(to_return)
        if errors:
            to_return['errors'] = errors
        if self.debug:
            print(json.dumps(to_return, indent=4))
        return to_return

    # ################################################
    # ############### Simple REST API ################
    # ################################################

    def get_index(self, filters=None):
        """
            Return the index.

            Warning, there's a limit on the number of results
        """
        session = self.__prepare_session()
        url = urljoin(self.root_url, 'events/index')
        if filters is not None:
            filters = json.dumps(filters)
            response = session.post(url, data=filters)
        else:
            response = session.get(url)
        return self._check_response(response)

    def get_event(self, event_id):
        """
            Get an event

            :param event_id: Event id to get
        """
        session = self.__prepare_session()
        url = urljoin(self.root_url, 'events/{}'.format(event_id))
        response = session.get(url)
        return self._check_response(response)

    def get_stix_event(self, event_id=None, with_attachments=False, from_date=False, to_date=False, tags=False):
        """
            Get an event/events in STIX format
        """
        if tags:
            if isinstance(tags, list):
                tags = "&&".join(tags)

        session = self.__prepare_session()
        url = urljoin(self.root_url, "/events/stix/download/{}/{}/{}/{}/{}".format(
            event_id, with_attachments, tags, from_date, to_date))
        if self.debug:
            print("Getting STIX event from {}".format(url))
        response = session.get(url)
        return self._check_response(response)

    def add_event(self, event):
        """
            Add a new event

            :param event: Event as JSON object / string or XML to add
        """
        session = self.__prepare_session()
        url = urljoin(self.root_url, 'events')
        if isinstance(event, basestring):
            response = session.post(url, data=event)
        else:
            response = session.post(url, data=json.dumps(event))
        return self._check_response(response)

    def update_event(self, event_id, event):
        """
            Update an event

            :param event_id: Event id to update
            :param event: Event as JSON object / string or XML to add
        """
        session = self.__prepare_session()
        url = urljoin(self.root_url, 'events/{}'.format(event_id))
        if isinstance(event, basestring):
            response = session.post(url, data=event)
        else:
            response = session.post(url, data=json.dumps(event))
        return self._check_response(response)

    def delete_event(self, event_id):
        """
            Delete an event

            :param event_id: Event id to delete
        """
        session = self.__prepare_session()
        url = urljoin(self.root_url, 'events/{}'.format(event_id))
        response = session.delete(url)
        return self._check_response(response)

    def delete_attribute(self, attribute_id):
        session = self.__prepare_session()
        url = urljoin(self.root_url, 'attributes/{}'.format(attribute_id))
        response = session.delete(url)
        return self._check_response(response)

    # ##############################################
    # ######### Event handling (Json only) #########
    # ##############################################

    def _prepare_full_event(self, distribution, threat_level_id, analysis, info, date=None, published=False):
        to_return = {'Event': {}}
        # Setup details of a new event
        if distribution not in [0, 1, 2, 3]:
            raise NewEventError('{} is invalid, the distribution has to be in 0, 1, 2, 3'.format(distribution))
        if threat_level_id not in [1, 2, 3, 4]:
            raise NewEventError('{} is invalid, the threat_level_id has to be in 1, 2, 3, 4'.format(threat_level_id))
        if analysis not in [0, 1, 2]:
            raise NewEventError('{} is invalid, the analysis has to be in 0, 1, 2'.format(analysis))
        if date is None:
            date = datetime.date.today().isoformat()
        if published not in [True, False]:
            raise NewEventError('{} is invalid, published has to be True or False'.format(published))
        to_return['Event'] = {'distribution': distribution, 'info': info, 'date': date, 'published': published,
                              'threat_level_id': threat_level_id, 'analysis': analysis}
        return to_return

    def _prepare_full_attribute(self, category, type_value, value, to_ids, comment=None, distribution=None):
        to_return = {}
        if category not in self.categories:
            raise NewAttributeError('{} is invalid, category has to be in {}'.format(category, (', '.join(self.categories))))

        if type_value not in self.types:
            raise NewAttributeError('{} is invalid, type_value has to be in {}'.format(type_value, (', '.join(self.types))))

        if type_value not in self.category_type_mapping[category]:
            raise NewAttributeError('{} and {} is an invalid combinaison, type_value for this category has to be in {}'.format(type_value, category, (', '.join(self.category_type_mapping[category]))))
        to_return['type'] = type_value
        to_return['category'] = category

        if to_ids not in [True, False]:
            raise NewAttributeError('{} is invalid, to_ids has to be True or False'.format(to_ids))
        to_return['to_ids'] = to_ids

        if distribution is not None:
            distribution = int(distribution)
        # If None: take the default value of the event
        if distribution not in [None, 0, 1, 2, 3, 5]:
            raise NewAttributeError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 5 or None'.format(distribution))
        if distribution is not None:
            to_return['distribution'] = distribution

        to_return['value'] = value

        if comment is not None:
            to_return['comment'] = comment

        return to_return

    def _prepare_update(self, event):
        # Cleanup the received event to make it publishable
        event['Event'].pop('locked', None)
        event['Event'].pop('attribute_count', None)
        event['Event'].pop('RelatedEvent', None)
        event['Event'].pop('orgc', None)
        event['Event'].pop('ShadowAttribute', None)
        event['Event'].pop('org', None)
        event['Event'].pop('proposal_email_lock', None)
        event['Event'].pop('publish_timestamp', None)
        event['Event'].pop('published', None)
        event['Event'].pop('timestamp', None)
        event['Event']['id'] = int(event['Event']['id'])
        return event

    def _one_or_more(self, value):
        """Returns a list/tuple of one or more items, regardless of input."""
        return value if isinstance(value, (tuple, list)) else (value,)

    # ########## Helpers ##########

    def get(self, eid):
        response = self.get_event(int(eid))
        return response

    def get_stix(self, **kwargs):
        response = self.get_stix_event(**kwargs)
        return response

    def update(self, event):
        eid = event['Event']['id']
        response = self.update_event(eid, event)
        return response

    def new_event(self, distribution=None, threat_level_id=None, analysis=None, info=None, date=None, published=False):
        data = self._prepare_full_event(distribution, threat_level_id, analysis, info, date, published)
        response = self.add_event(data)
        return response

    def publish(self, event):
        if event['Event']['published']:
            return {'error': 'Already published'}
        event = self._prepare_update(event)
        event['Event']['published'] = True
        response = self.update_event(event['Event']['id'], event)
        return response

    def add_tag(self, event, tag):
        session = self.__prepare_session()
        to_post = {'request': {'Event': {'id': event['Event']['id'], 'tag': tag}}}
        response = session.post(urljoin(self.root_url, 'events/addTag'), data=json.dumps(to_post))
        return self._check_response(response)

    def remove_tag(self, event, tag):
        session = self.__prepare_session()
        to_post = {'request': {'Event': {'id': event['Event']['id'], 'tag': tag}}}
        response = session.post(urljoin(self.root_url, 'events/removeTag'), data=json.dumps(to_post))
        return self._check_response(response)

    def change_threat_level(self, event, threat_level_id):
        event['Event']['threat_level_id'] = threat_level_id
        self._prepare_update(event)
        response = self.update_event(event['Event']['id'], event)
        return response

    # ##### File attributes #####

    def _send_attributes(self, event, attributes, proposal=False):
        if proposal:
            response = self.proposal_add(event['Event']['id'], attributes)
        else:
            event = self._prepare_update(event)
            for a in attributes:
                if a.get('distribution') is None:
                    a['distribution'] = 5
            event['Event']['Attribute'] = attributes
            response = self.update_event(event['Event']['id'], event)
        return response

    def add_named_attribute(self, event, category, type_value, value, to_ids=False, comment=None, distribution=None, proposal=False):
        attributes = []
        if value and category and type:
            try:
                attributes.append(self._prepare_full_attribute(category, type_value, value, to_ids, comment, distribution))
            except NewAttributeError as e:
                return e
        return self._send_attributes(event, attributes, proposal)

    def add_hashes(self, event, category='Artifacts dropped', filename=None, md5=None, sha1=None, sha256=None, ssdeep=None, comment=None, to_ids=True, distribution=None, proposal=False):

        attributes = []
        type_value = '{}'
        value = '{}'
        if filename:
            type_value = 'filename|{}'
            value = filename + '|{}'
        if md5:
            attributes.append(self._prepare_full_attribute(category, type_value.format('md5'), value.format(md5),
                                                           to_ids, comment, distribution))
        if sha1:
            attributes.append(self._prepare_full_attribute(category, type_value.format('sha1'), value.format(sha1),
                                                           to_ids, comment, distribution))
        if sha256:
            attributes.append(self._prepare_full_attribute(category, type_value.format('sha256'), value.format(sha256),
                                                           to_ids, comment, distribution))
        if ssdeep:
            attributes.append(self._prepare_full_attribute(category, type_value.format('ssdeep'), value.format(ssdeep),
                                                           to_ids, comment, distribution))

        return self._send_attributes(event, attributes, proposal)

    def av_detection_link(self, event, link, category='Antivirus detection', to_ids=False, comment=None, distribution=None, proposal=False):
        attributes = []
        for link in self._one_or_more(link):
            attributes.append(self._prepare_full_attribute(category, 'link', link, to_ids, comment, distribution))
        return self._send_attributes(event, attributes, proposal)

    def add_detection_name(self, event, name, category='Antivirus detection', to_ids=False, comment=None, distribution=None, proposal=False):
        attributes = []
        for name in self._one_or_more(name):
            attributes.append(self._prepare_full_attribute(category, 'text', name, to_ids, comment, distribution))
        return self._send_attributes(event, attributes, proposal)

    def add_filename(self, event, filename, category='Artifacts dropped', to_ids=False, comment=None, distribution=None, proposal=False):
        attributes = []
        for filename in self._one_or_more(filename):
            attributes.append(self._prepare_full_attribute(category, 'filename', filename, to_ids, comment, distribution))
        return self._send_attributes(event, attributes, proposal)

    def add_regkey(self, event, regkey, rvalue=None, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False):
        if rvalue:
            type_value = 'regkey|value'
            value = '{}|{}'.format(regkey, rvalue)
        else:
            type_value = 'regkey'
            value = regkey

        attributes = []
        attributes.append(self._prepare_full_attribute(category, type_value, value, to_ids, comment, distribution))
        return self._send_attributes(event, attributes, proposal)

    def add_regkeys(self, event, regkeys_values, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False):
        attributes = []

        for regkey, rvalue in regkeys_values.items():
            if rvalue:
                type_value = 'regkey|value'
                value = '{}|{}'.format(regkey, rvalue)
            else:
                type_value = 'regkey'
                value = regkey

            attributes.append(self._prepare_full_attribute(category, type_value, value, to_ids, comment, distribution))
        return self._send_attributes(event, attributes, proposal)

    def add_pattern(self, event, pattern, in_file=True, in_memory=False, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False):
        attributes = []
        for pattern in self._one_or_more(pattern):
            if in_file:
                attributes.append(self._prepare_full_attribute(category, 'pattern-in-file', pattern, to_ids, comment, distribution))
            if in_memory:
                attributes.append(self._prepare_full_attribute(category, 'pattern-in-memory', pattern, to_ids, comment, distribution))

        return self._send_attributes(event, attributes, proposal)

    def add_pipe(self, event, named_pipe, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False):
        attributes = []
        for named_pipe in self._one_or_more(named_pipe):
            if not named_pipe.startswith('\\.\\pipe\\'):
                named_pipe = '\\.\\pipe\\{}'.format(named_pipe)
            attributes.append(self._prepare_full_attribute(category, 'named pipe', named_pipe, to_ids, comment, distribution))
        return self._send_attributes(event, attributes, proposal)

    def add_mutex(self, event, mutex, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False):
        attributes = []
        if not mutex.startswith('\\BaseNamedObjects\\'):
            mutex = '\\BaseNamedObjects\\{}'.format(mutex)
        attributes.append(self._prepare_full_attribute(category, 'mutex', mutex, to_ids, comment, distribution))
        return self._send_attributes(event, attributes, proposal)

    def add_yara(self, event, yara, category='Payload delivery', to_ids=False, comment=None, distribution=None, proposal=False):
        attributes = []
        for yara in self._one_or_more(yara):
            attributes.append(self._prepare_full_attribute(category, 'yara', yara, to_ids, comment, distribution))
        return self._send_attributes(event, attributes, proposal)

    # ##### Network attributes #####

    def add_ipdst(self, event, ipdst, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False):
        attributes = []
        for ipdst in self._one_or_more(ipdst):
            attributes.append(self._prepare_full_attribute(category, 'ip-dst', ipdst, to_ids, comment, distribution))
        return self._send_attributes(event, attributes, proposal)

    def add_ipsrc(self, event, ipsrc, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False):
        attributes = []
        for ipsrc in self._one_or_more(ipsrc):
            attributes.append(self._prepare_full_attribute(category, 'ip-src', ipsrc, to_ids, comment, distribution))
        return self._send_attributes(event, attributes, proposal)

    def add_hostname(self, event, hostname, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False):
        attributes = []
        for hostname in self._one_or_more(hostname):
            attributes.append(self._prepare_full_attribute(category, 'hostname', hostname, to_ids, comment, distribution))
        return self._send_attributes(event, attributes, proposal)

    def add_domain(self, event, domain, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False):
        attributes = []
        for domain in self._one_or_more(domain):
            attributes.append(self._prepare_full_attribute(category, 'domain', domain, to_ids, comment, distribution))
        return self._send_attributes(event, attributes, proposal)

    def add_domain_ip(self, event, domain, ip, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False):
        attributes = []
        attributes.append(self._prepare_full_attribute(category, 'domain|ip', "%s|%s" % (domain, ip), to_ids, comment, distribution))
        return self._send_attributes(event, attributes, proposal)

    def add_domains_ips(self, event, domain_ips, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False):
        attributes = []
        for domain, ip in domain_ips.items():
            attributes.append(self._prepare_full_attribute(category, 'domain|ip', "%s|%s" % (domain, ip), to_ids, comment, distribution))
        return self._send_attributes(event, attributes, proposal)

    def add_url(self, event, url, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False):
        attributes = []
        for url in self._one_or_more(url):
            attributes.append(self._prepare_full_attribute(category, 'url', url, to_ids, comment, distribution))
        return self._send_attributes(event, attributes, proposal)

    def add_useragent(self, event, useragent, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False):
        attributes = []
        for useragent in self._one_or_more(useragent):
            attributes.append(self._prepare_full_attribute(category, 'user-agent', useragent, to_ids, comment, distribution))
        return self._send_attributes(event, attributes, proposal)

    def add_traffic_pattern(self, event, pattern, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False):
        attributes = []
        for pattern in self._one_or_more(pattern):
            attributes.append(self._prepare_full_attribute(category, 'pattern-in-traffic', pattern, to_ids, comment, distribution))
        return self._send_attributes(event, attributes, proposal)

    def add_snort(self, event, snort, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False):
        attributes = []
        for snort in self._one_or_more(snort):
            attributes.append(self._prepare_full_attribute(category, 'snort', snort, to_ids, comment, distribution))
        return self._send_attributes(event, attributes, proposal)

    # ##### Email attributes #####

    def add_email_src(self, event, email, to_ids=True, comment=None, distribution=None, proposal=False):
        attributes = []
        for email in self._one_or_more(email):
            attributes.append(self._prepare_full_attribute('Payload delivery', 'email-src', email, to_ids, comment, distribution))
        return self._send_attributes(event, attributes, proposal)

    def add_email_dst(self, event, email, category='Payload delivery', to_ids=True, comment=None, distribution=None, proposal=False):
        attributes = []
        for email in self._one_or_more(email):
            attributes.append(self._prepare_full_attribute(category, 'email-dst', email, to_ids, comment, distribution))
        return self._send_attributes(event, attributes, proposal)

    def add_email_subject(self, event, email, to_ids=True, comment=None, distribution=None, proposal=False):
        attributes = []
        for email in self._one_or_more(email):
            attributes.append(self._prepare_full_attribute('Payload delivery', 'email-subject', email, to_ids, comment, distribution))
        return self._send_attributes(event, attributes, proposal)

    def add_email_attachment(self, event, email, to_ids=True, comment=None, distribution=None, proposal=False):
        attributes = []
        for email in self._one_or_more(email):
            attributes.append(self._prepare_full_attribute('Payload delivery', 'email-attachment', email, to_ids, comment, distribution))
        return self._send_attributes(event, attributes, proposal)

    # ##### Target attributes #####

    def add_target_email(self, event, target, to_ids=True, comment=None, distribution=None, proposal=False):
        attributes = []
        for target in self._one_or_more(target):
            attributes.append(self._prepare_full_attribute('Targeting data', 'target-email', target, to_ids, comment, distribution))
        return self._send_attributes(event, attributes, proposal)

    def add_target_user(self, event, target, to_ids=True, comment=None, distribution=None, proposal=False):
        attributes = []
        for target in self._one_or_more(target):
            attributes.append(self._prepare_full_attribute('Targeting data', 'target-user', target, to_ids, comment, distribution))
        return self._send_attributes(event, attributes, proposal)

    def add_target_machine(self, event, target, to_ids=True, comment=None, distribution=None, proposal=False):
        attributes = []
        for target in self._one_or_more(target):
            attributes.append(self._prepare_full_attribute('Targeting data', 'target-machine', target, to_ids, comment, distribution))
        return self._send_attributes(event, attributes, proposal)

    def add_target_org(self, event, target, to_ids=True, comment=None, distribution=None, proposal=False):
        attributes = []
        for target in self._one_or_more(target):
            attributes.append(self._prepare_full_attribute('Targeting data', 'target-org', target, to_ids, comment, distribution))
        return self._send_attributes(event, attributes, proposal)

    def add_target_location(self, event, target, to_ids=True, comment=None, distribution=None, proposal=False):
        attributes = []
        for target in self._one_or_more(target):
            attributes.append(self._prepare_full_attribute('Targeting data', 'target-location', target, to_ids, comment, distribution))
        return self._send_attributes(event, attributes, proposal)

    def add_target_external(self, event, target, to_ids=True, comment=None, distribution=None, proposal=False):
        attributes = []
        for target in self._one_or_more(target):
            attributes.append(self._prepare_full_attribute('Targeting data', 'target-external', target, to_ids, comment, distribution))
        return self._send_attributes(event, attributes, proposal)

    # ##### Attribution attributes #####

    def add_threat_actor(self, event, target, to_ids=True, comment=None, distribution=None, proposal=False):
        attributes = []
        for target in self._one_or_more(target):
            attributes.append(self._prepare_full_attribute('Attribution', 'threat-actor', target, to_ids, comment, distribution))
        return self._send_attributes(event, attributes, proposal)

    # ##### Internal reference attributes #####

    def add_internal_link(self, event, reference, to_ids=False, comment=None, distribution=None, proposal=False):
        attributes = []
        for reference in self._one_or_more(reference):
            attributes.append(self._prepare_full_attribute('Internal reference', 'link', reference, to_ids, comment, distribution))
        return self._send_attributes(event, attributes, proposal)

    def add_internal_comment(self, event, reference, to_ids=False, comment=None, distribution=None, proposal=False):
        attributes = []
        for reference in self._one_or_more(reference):
            attributes.append(self._prepare_full_attribute('Internal reference', 'comment', reference, to_ids, comment, distribution))
        return self._send_attributes(event, attributes, proposal)

    def add_internal_text(self, event, reference, to_ids=False, comment=None, distribution=None, proposal=False):
        attributes = []
        for reference in self._one_or_more(reference):
            attributes.append(self._prepare_full_attribute('Internal reference', 'text', reference, to_ids, comment, distribution))
        return self._send_attributes(event, attributes, proposal)

    def add_internal_other(self, event, reference, to_ids=False, comment=None, distribution=None, proposal=False):
        attributes = []
        for reference in self._one_or_more(reference):
            attributes.append(self._prepare_full_attribute('Internal reference', 'other', reference, to_ids, comment, distribution))
        return self._send_attributes(event, attributes, proposal)

    # ##################################################
    # ######### Upload samples through the API #########
    # ##################################################

    def _create_event(self, distribution, threat_level_id, analysis, info):
        # Setup details of a new event
        if distribution not in [0, 1, 2, 3]:
            raise NewEventError('{} is invalid, the distribution has to be in 0, 1, 2, 3'.format(distribution))
        if threat_level_id not in [1, 2, 3, 4]:
            raise NewEventError('{} is invalid, the threat_level_id has to be in 1, 2, 3, 4'.format(threat_level_id))
        if analysis not in [0, 1, 2]:
            raise NewEventError('{} is invalid, the analysis has to be in 0, 1, 2'.format(analysis))
        return {'distribution': int(distribution), 'info': info,
                'threat_level_id': int(threat_level_id), 'analysis': analysis}

    def prepare_attribute(self, event_id, distribution, to_ids, category, comment, info,
                          analysis, threat_level_id):
        to_post = {'request': {}}
        authorized_categs = ['Payload delivery', 'Artifacts dropped', 'Payload installation', 'External analysis', 'Network activity', 'Antivirus detection']

        if event_id is not None:
            try:
                event_id = int(event_id)
            except:
                pass
        if not isinstance(event_id, int):
            # New event
            to_post['request'] = self._create_event(distribution, threat_level_id, analysis, info)
        else:
            to_post['request']['event_id'] = int(event_id)

        if to_ids not in [True, False]:
            raise NewAttributeError('{} is invalid, to_ids has to be True or False'.format(to_ids))
        to_post['request']['to_ids'] = to_ids

        if category not in authorized_categs:
            raise NewAttributeError('{} is invalid, category has to be in {}'.format(category, (', '.join(authorized_categs))))
        to_post['request']['category'] = category

        to_post['request']['comment'] = comment
        return to_post

    def _encode_file_to_upload(self, path):
        with open(path, 'rb') as f:
            return str(base64.b64encode(f.read()))

    def upload_sample(self, filename, filepath, event_id, distribution=None,
                      to_ids=True, category=None, comment=None, info=None,
                      analysis=None, threat_level_id=None):
        to_post = self.prepare_attribute(event_id, distribution, to_ids, category,
                                         comment, info, analysis, threat_level_id)
        to_post['request']['files'] = [{'filename': filename, 'data': self._encode_file_to_upload(filepath)}]
        return self._upload_sample(to_post)

    def upload_samplelist(self, filepaths, event_id, distribution=None,
                          to_ids=True, category=None, info=None,
                          analysis=None, threat_level_id=None):
        to_post = self.prepare_attribute(event_id, distribution, to_ids, category,
                                         info, analysis, threat_level_id)
        files = []
        for path in filepaths:
            if not os.path.isfile(path):
                continue
            files.append({'filename': os.path.basename(path), 'data': self._encode_file_to_upload(path)})
        to_post['request']['files'] = files
        return self._upload_sample(to_post)

    def _upload_sample(self, to_post):
        session = self.__prepare_session()
        url = urljoin(self.root_url, 'events/upload_sample')
        response = session.post(url, data=json.dumps(to_post))
        return self._check_response(response)

    # ############################
    # ######## Proposals #########
    # ############################

    def __query_proposal(self, session, path, id, attribute=None):
        path = path.strip('/')
        url = urljoin(self.root_url, 'shadow_attributes/{}/{}'.format(path, id))
        query = None
        if path in ['add', 'edit']:
            query = {'request': {'ShadowAttribute': attribute}}
        if path == 'view':
            response = session.get(url)
        else:
            if query is not None:
                response = session.post(url, data=json.dumps(query))
            else:
                response = session.post(url)
        return self._check_response(response)

    def proposal_view(self, event_id=None, proposal_id=None):
        session = self.__prepare_session()
        if proposal_id is not None and event_id is not None:
            return {'error': 'You can only view an event ID or a proposal ID'}
        if event_id is not None:
            id = event_id
        else:
            id = proposal_id
        return self.__query_proposal(session, 'view', id)

    def proposal_add(self, event_id, attribute):
        session = self.__prepare_session()
        return self.__query_proposal(session, 'add', event_id, attribute)

    def proposal_edit(self, attribute_id, attribute):
        session = self.__prepare_session()
        return self.__query_proposal(session, 'edit', attribute_id, attribute)

    def proposal_accept(self, proposal_id):
        session = self.__prepare_session()
        return self.__query_proposal(session, 'accept', proposal_id)

    def proposal_discard(self, proposal_id):
        session = self.__prepare_session()
        return self.__query_proposal(session, 'discard', proposal_id)

    # ##############################
    # ######## REST Search #########
    # ##############################

    def __query(self, session, path, query, controller='events'):
        if query.get('error') is not None:
            return query
        if controller not in ['events', 'attributes']:
            raise Exception('Invalid controller. Can only be {}'.format(', '.join(['events', 'attributes'])))
        url = urljoin(self.root_url, '{}/{}'.format(controller, path.lstrip('/')))
        query = {'request': query}
        response = session.post(url, data=json.dumps(query))
        return self._check_response(response)

    def search_index(self, published=None, eventid=None, tag=None, datefrom=None,
                     dateto=None, eventinfo=None, threatlevel=None, distribution=None,
                     analysis=None, attribute=None, org=None):
        """
            Search only at the index level. Use ! infront of value as NOT, default OR

            :param published: Published (0,1)
            :param eventid: Evend ID(s) | str or list
            :param tag: Tag(s) | str or list
            :param datefrom: First date, in format YYYY-MM-DD
            :param datefrom: Last date, in format YYYY-MM-DD
            :param eventinfo: Event info(s) to match | str or list
            :param threatlevel: Threat level(s) (1,2,3,4) | str or list
            :param distribution: Distribution level(s) (0,1,2,3) | str or list
            :param analysis: Analysis level(s) (0,1,2) | str or list
            :param org: Organisation(s) | str or list

        """
        allowed = {'published': published, 'eventid': eventid, 'tag': tag, 'Dateto': dateto,
                   'Datefrom': datefrom, 'eventinfo': eventinfo, 'threatlevel': threatlevel,
                   'distribution': distribution, 'analysis': analysis, 'attribute': attribute,
                   'org': org}
        rule_levels = {'distribution': ["0", "1", "2", "3", "!0", "!1", "!2", "!3"],
                       'threatlevel': ["1", "2", "3", "4", "!1", "!2", "!3", "!4"],
                       'analysis': ["0", "1", "2", "!0", "!1", "!2"]}
        buildup_url = "events/index"

        for rule in allowed.keys():
            if allowed[rule] is not None:
                if not isinstance(allowed[rule], list):
                    allowed[rule] = [allowed[rule]]
                allowed[rule] = map(str, allowed[rule])
                if rule in rule_levels:
                    if not set(allowed[rule]).issubset(rule_levels[rule]):
                        raise SearchError('Values in your {} are invalid, has to be in {}'.format(rule, ', '.join(str(x) for x in rule_levels[rule])))
                if type(allowed[rule]) == list:
                    joined = '|'.join(str(x) for x in allowed[rule])
                    buildup_url += '/search{}:{}'.format(rule, joined)
                else:
                    buildup_url += '/search{}:{}'.format(rule, allowed[rule])
        session = self.__prepare_session()
        url = urljoin(self.root_url, buildup_url)
        response = session.get(url)
        return self._check_response(response)

    def search_all(self, value):
        query = {'value': value, 'searchall': 1}
        session = self.__prepare_session()
        return self.__query(session, 'restSearch/download', query)

    def __prepare_rest_search(self, values, not_values):
        """
            Prepare a search, generate the chain processed by the server

            :param values: Values to search
            :param not_values: Values that should not be in the response
        """
        to_return = ''
        if values is not None:
            if not isinstance(values, list):
                to_return += values
            else:
                to_return += '&&'.join(values)
        if not_values is not None:
            if len(to_return) > 0:
                to_return += '&&!'
            else:
                to_return += '!'
            if not isinstance(values, list):
                to_return += not_values
            else:
                to_return += '&&!'.join(not_values)
        return to_return

    def search(self, values=None, not_values=None, type_attribute=None,
               category=None, org=None, tags=None, not_tags=None, date_from=None,
               date_to=None, last=None, controller='events'):
        """
            Search via the Rest API

            :param values: values to search for
            :param not_values: values *not* to search for
            :param type_attribute: Type of attribute
            :param category: Category to search
            :param org: Org reporting the event
            :param tags: Tags to search for
            :param not_tags: Tags *not* to search for
            :param date_from: First date
            :param date_to: Last date
            :param last: Last updated events (for example 5d or 12h or 30m)

        """
        val = self.__prepare_rest_search(values, not_values).replace('/', '|')
        tag = self.__prepare_rest_search(tags, not_tags).replace(':', ';')
        query = {}
        if len(val) != 0:
            query['value'] = val
        if len(tag) != 0:
            query['tags'] = tag
        if type_attribute is not None:
            query['type'] = type_attribute
        if category is not None:
            query['category'] = category
        if org is not None:
            query['org'] = org
        if date_from is not None:
            if isinstance(date_from, datetime.date) or isinstance(date_to, datetime.datetime):
                query['from'] = date_from.strftime('%Y-%m-%d')
            else:
                query['from'] = date_from
        if date_to is not None:
            if isinstance(date_to, datetime.date) or isinstance(date_to, datetime.datetime):
                query['to'] = date_to.strftime('%Y-%m-%d')
            else:
                query['to'] = date_to
        if last is not None:
            query['last'] = last

        session = self.__prepare_session()
        return self.__query(session, 'restSearch/download', query, controller)

    def get_attachement(self, event_id):
        """
            Get attachement of an event (not sample)

            :param event_id: Event id from where the attachements will
                             be fetched
        """
        attach = urljoin(self.root_url, 'attributes/downloadAttachment/download/{}'.format(event_id))
        session = self.__prepare_session()
        response = session.get(attach)
        return self._check_response(response)

    def get_yara(self, event_id):
        to_post = {'request': {'eventid': event_id, 'type': 'yara'}}
        session = self.__prepare_session()
        response = session.post(urljoin(self.root_url, 'attributes/restSearch'), data=json.dumps(to_post))
        result = self._check_response(response)
        if result.get('error') is not None:
            return False, result.get('error')
        if not result.get('response'):
            return False, result.get('message')
        rules = '\n\n'.join([a['value'] for a in result['response']['Attribute']])
        return True, rules

    def download_samples(self, sample_hash=None, event_id=None, all_samples=False):
        to_post = {'request': {'hash': sample_hash, 'eventID': event_id, 'allSamples': all_samples}}
        session = self.__prepare_session()
        response = session.post(urljoin(self.root_url, 'attributes/downloadSample'), data=json.dumps(to_post))
        result = self._check_response(response)
        if result.get('error') is not None:
            return False, result.get('error')
        if not result.get('result'):
            return False, result.get('message')
        details = []
        for f in result['result']:
            decoded = base64.b64decode(f['base64'])
            zipped = BytesIO(decoded)
            try:
                archive = zipfile.ZipFile(zipped)
                try:
                    # New format
                    unzipped = BytesIO(archive.open(f['md5'], pwd=b'infected').read())
                except KeyError:
                    # Old format
                    unzipped = BytesIO(archive.open(f['filename'], pwd=b'infected').read())
                details.append([f['event_id'], f['filename'], unzipped])
            except zipfile.BadZipfile:
                # In case the sample isn't zipped
                details.append([f['event_id'], f['filename'], zipped])

        return True, details

    def download_last(self, last):
        """
            Download the last updated events.

            :param last: can be defined in days, hours, minutes (for example 5d or 12h or 30m)
        """
        return self.search(last=last)

    # ############## Suricata ###############

    def download_all_suricata(self):
        """
            Download all suricata rules events.
        """
        suricata_rules = urljoin(self.root_url, 'events/nids/suricata/download')
        session = self.__prepare_session('rules')
        response = session.get(suricata_rules)
        return response

    def download_suricata_rule_event(self, event_id):
        """
            Download one suricata rule event.

            :param event_id: ID of the event to download (same as get)
        """
        template = urljoin(self.root_url, 'events/nids/suricata/download/{}'.format(event_id))
        session = self.__prepare_session('rules')
        response = session.get(template)
        return response

    # ########## Tags ##########

    def get_all_tags(self, quiet=False):
        session = self.__prepare_session()
        url = urljoin(self.root_url, 'tags')
        response = session.get(url)
        r = self._check_response(response)
        if not quiet or r.get('errors'):
            return r
        else:
            to_return = []
            for tag in r['Tag']:
                to_return.append(tag['name'])
            return to_return

    def new_tag(self, name=None, colour="#00ace6", exportable=False):
        to_post = {'Tag': {'name': name, 'colour': colour, 'exportable': exportable}}
        session = self.__prepare_session()
        url = urljoin(self.root_url, 'tags/add')
        response = session.post(url, data=json.dumps(to_post))
        return self._check_response(response)

    # ########## Version ##########

    def get_api_version(self):
        """
            Returns the current version of PyMISP installed on the system
        """
        return {'version': __version__}

    def get_api_version_master(self):
        """
            Get the most recent version of PyMISP from github
        """
        r = requests.get('https://raw.githubusercontent.com/MISP/PyMISP/master/pymisp/__init__.py')
        if r.status_code == 200:
            version = re.findall("__version__ = '(.*)'", r.text)
            return {'version': version[0]}
        else:
            return {'error': 'Impossible to retrieve the version of the master branch.'}

    def get_version(self):
        """
            Returns the version of the instance.
        """
        session = self.__prepare_session()
        url = urljoin(self.root_url, 'servers/getVersion.json')
        response = session.get(url)
        return self._check_response(response)

    def get_version_master(self):
        """
            Get the most recent version from github
        """
        r = requests.get('https://raw.githubusercontent.com/MISP/MISP/2.4/VERSION.json')
        if r.status_code == 200:
            master_version = json.loads(r.text)
            return {'version': '{}.{}.{}'.format(master_version['major'], master_version['minor'], master_version['hotfix'])}
        else:
            return {'error': 'Impossible to retrieve the version of the master branch.'}
    # ############## Export Attributes in text ####################################

    def get_all_attributes_txt(self, type_attr):

        session = self.__prepare_session('txt')
        url = urljoin(self.root_url, 'attributes/text/download/%s' % type_attr)
        response = session.get(url)
        return response

    # ############## Statistics ##################

    def get_attributes_statistics(self, context='type', percentage=None):
        """
            Get attributes statistics from the MISP instance
        """
        session = self.__prepare_session()
        if (context != 'category'):
            context = 'type'
        if percentage is not None:
            url = urljoin(self.root_url, 'attributes/attributeStatistics/{}/{}'.format(context, percentage))
        else:
            url = urljoin(self.root_url, 'attributes/attributeStatistics/{}'.format(context))
        response = session.get(url)
        return self._check_response(response)

    def get_tags_statistics(self, percentage=None, name_sort=None):
        """
        Get tags statistics from the MISP instance
        """
        session = self.__prepare_session()
        if percentage is not None:
            percentage = 'true'
        else:
            percentage = 'false'
        if name_sort is not None:
            name_sort = 'true'
        else:
            name_sort = 'false'
        url = urljoin(self.root_url, 'tags/tagStatistics/{}/{}'.format(percentage, name_sort))
        response = session.get(url).json()
        return self._check_response(response)

    # ############## Sightings ##################

    def sighting_per_id(self, attribute_id):
        session = self.__prepare_session()
        url = urljoin(self.root_url, 'sightings/add/{}'.format(attribute_id))
        response = session.post(url)
        return self._check_response(response)

    def sighting_per_uuid(self, attribute_uuid):
        session = self.__prepare_session()
        url = urljoin(self.root_url, 'sightings/add/{}'.format(attribute_uuid))
        response = session.post(url)
        return self._check_response(response)

    def sighting_per_json(self, json_file):
        session = self.__prepare_session()
        jdata = json.load(open(json_file))
        url = urljoin(self.root_url, 'sightings/add/')
        response = session.post(url, data=json.dumps(jdata))
        return self._check_response(response)

    # ############## Sharing Groups ##################

    def get_sharing_groups(self):
        session = self.__prepare_session()
        url = urljoin(self.root_url, 'sharing_groups/index.json')
        response = session.get(url)
        return self._check_response(response)['response'][0]