diff --git a/.travis.yml b/.travis.yml index 8a6027c..31ce007 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,12 +12,6 @@ addons: matrix: include: - - name: "Python 2.7 - legacy" - python: 2.7 - env: LEGACY=true - - name: "Python 3.5" - python: 3.5 - dist: xenial - name: "Python 3.6" python: 3.6 dist: xenial diff --git a/Pipfile.lock b/Pipfile.lock index 8894250..403c70a 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -68,11 +68,11 @@ }, "importlib-metadata": { "hashes": [ - "sha256:b044f07694ef14a6683b097ba56bd081dbc7cdc7c7fe46011e499dfecc082f21", - "sha256:e6ac600a142cf2db707b1998382cc7fc3b02befb7273876e01b8ad10b9652742" + "sha256:073a852570f92da5f744a3472af1b61e28e9f78ccf0c9117658dc32b15de7b45", + "sha256:d95141fbfa7ef2ec65cfd945e2af7e5a6ddbd7c8d9a25e66ff3be8e3daf9f60f" ], "markers": "python_version < '3.8'", - "version": "==1.1.0" + "version": "==1.3.0" }, "jsonschema": { "hashes": [ @@ -102,10 +102,10 @@ }, "more-itertools": { "hashes": [ - "sha256:53ff73f186307d9c8ef17a9600309154a6ae27f25579e80af4db8f047ba14bc2", - "sha256:a0ea684c39bc4315ba7aae406596ef191fd84f873d2d2751f84d64e81a7a2d45" + "sha256:b84b238cce0d9adad5ed87e745778d20a3f8487d0f0cb8b8a586816c7496458d", + "sha256:c833ef592a0324bcc6a60e48440da07645063c453880c9477ceb22490aec1564" ], - "version": "==8.0.0" + "version": "==8.0.2" }, "pillow": { "hashes": [ @@ -246,9 +246,9 @@ }, "validators": { "hashes": [ - "sha256:f0ac832212e3ee2e9b10e156f19b106888cf1429c291fbc5297aae87685014ae" + "sha256:0bfe836a1af37bb266d71ec1e98b530c38ce11bc7fbe0c4c96ef7b1532d019e5" ], - "version": "==0.14.0" + "version": "==0.14.1" }, "wrapt": { "hashes": [ @@ -325,10 +325,10 @@ }, "colorama": { "hashes": [ - "sha256:05eed71e2e327246ad6b38c540c4a3117230b19679b875190486ddd2d721422d", - "sha256:f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48" + "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff", + "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1" ], - "version": "==0.4.1" + "version": "==0.4.3" }, "commonmark": { "hashes": [ @@ -376,11 +376,11 @@ }, "coveralls": { "hashes": [ - "sha256:9bc5a1f92682eef59f688a8f280207190d9a6afb84cef8f567fa47631a784060", - "sha256:fb51cddef4bc458de347274116df15d641a735d3f0a580a9472174e2e62f408c" + "sha256:25522a50cdf720d956601ca6ef480786e655ae2f0c94270c77e1a23d742de558", + "sha256:8e3315e8620bb6b3c6f3179a75f498e7179c93b3ddc440352404f941b1f70524" ], "index": "pypi", - "version": "==1.8.2" + "version": "==1.9.2" }, "decorator": { "hashes": [ @@ -426,11 +426,11 @@ }, "importlib-metadata": { "hashes": [ - "sha256:b044f07694ef14a6683b097ba56bd081dbc7cdc7c7fe46011e499dfecc082f21", - "sha256:e6ac600a142cf2db707b1998382cc7fc3b02befb7273876e01b8ad10b9652742" + "sha256:073a852570f92da5f744a3472af1b61e28e9f78ccf0c9117658dc32b15de7b45", + "sha256:d95141fbfa7ef2ec65cfd945e2af7e5a6ddbd7c8d9a25e66ff3be8e3daf9f60f" ], "markers": "python_version < '3.8'", - "version": "==1.1.0" + "version": "==1.3.0" }, "jinja2": { "hashes": [ @@ -507,10 +507,10 @@ }, "more-itertools": { "hashes": [ - "sha256:53ff73f186307d9c8ef17a9600309154a6ae27f25579e80af4db8f047ba14bc2", - "sha256:a0ea684c39bc4315ba7aae406596ef191fd84f873d2d2751f84d64e81a7a2d45" + "sha256:b84b238cce0d9adad5ed87e745778d20a3f8487d0f0cb8b8a586816c7496458d", + "sha256:c833ef592a0324bcc6a60e48440da07645063c453880c9477ceb22490aec1564" ], - "version": "==8.0.0" + "version": "==8.0.2" }, "neobolt": { "hashes": [ @@ -740,10 +740,10 @@ }, "sphinx": { "hashes": [ - "sha256:31088dfb95359384b1005619827eaee3056243798c62724fd3fa4b84ee4d71bd", - "sha256:52286a0b9d7caa31efee301ec4300dbdab23c3b05da1c9024b4e84896fb73d79" + "sha256:0a11e2fd31fe5c7e64b4fc53c2c022946512f021d603eb41ac6ae51d5fcbb574", + "sha256:138e39aa10f28d52aa5759fc6d1cba2be6a4b750010974047fa7d0e31addcf63" ], - "version": "==2.2.1" + "version": "==2.3.0" }, "sphinx-autodoc-typehints": { "hashes": [ @@ -803,9 +803,9 @@ }, "validators": { "hashes": [ - "sha256:f0ac832212e3ee2e9b10e156f19b106888cf1429c291fbc5297aae87685014ae" + "sha256:0bfe836a1af37bb266d71ec1e98b530c38ce11bc7fbe0c4c96ef7b1532d019e5" ], - "version": "==0.14.0" + "version": "==0.14.1" }, "wcwidth": { "hashes": [ diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 3fa1c7d..bd78870 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -13,24 +13,18 @@ logger.addHandler(default_handler) logger.setLevel(logging.WARNING) -def warning_2020(): - - if sys.version_info < (3, 6): - warnings.warn(""" -Python 2.7 is officially end of life the 2020-01-01. For this occasion, -we decided to review which versions of Python we support and our conclusion -is to only support python 3.6+ starting the 2020-01-01. - -Every version of pymisp released after the 2020-01-01 will fail if the -python interpreter is prior to python 3.6. - -**Please update your codebase.**""", DeprecationWarning, stacklevel=3) +everything_broken = '''Unknown error: the response is not in JSON. +Something is broken server-side, please send us everything that follows (careful with the auth key): +Request headers: +{} +Request body: +{} +Response (if any): +{}''' try: - warning_2020() from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate, PyMISPInvalidFormat, MISPServerError, PyMISPNotImplementedYet, PyMISPUnexpectedResponse, PyMISPEmptyResponse # noqa - from .api import PyMISP # noqa from .abstract import AbstractMISP, MISPEncode, pymisp_json_default, MISPTag, Distribution, ThreatLevel, Analysis # noqa from .mispevent import MISPEvent, MISPAttribute, MISPObjectReference, MISPObjectAttribute, MISPObject, MISPUser, MISPOrganisation, MISPSighting, MISPLog, MISPShadowAttribute, MISPWarninglist, MISPTaxonomy, MISPNoticelist, MISPObjectTemplate, MISPSharingGroup, MISPRole, MISPServer, MISPFeed, MISPEventDelegation, MISPUserSetting # noqa from .tools import AbstractMISPObjectGenerator # noqa @@ -39,18 +33,18 @@ try: from .tools import openioc # noqa from .tools import ext_lookups # noqa - if sys.version_info >= (3, 6): - from .aping import ExpandedPyMISP # noqa - from .tools import load_warninglists # noqa - # Let's not bother with old python - try: - from .tools import reportlab_generator # noqa - except ImportError: - # FIXME: The import should not raise an exception if reportlab isn't installed - pass - except NameError: - # FIXME: The import should not raise an exception if reportlab isn't installed - pass + from .api import PyMISP # noqa + from .api import PyMISP as ExpandedPyMISP # noqa + from .tools import load_warninglists # noqa + # Let's not bother with old python + try: + from .tools import reportlab_generator # noqa + except ImportError: + # FIXME: The import should not raise an exception if reportlab isn't installed + pass + except NameError: + # FIXME: The import should not raise an exception if reportlab isn't installed + pass logger.debug('pymisp loaded properly') except ImportError as e: logger.warning('Unable to load pymisp properly: {}'.format(e)) diff --git a/pymisp/abstract.py b/pymisp/abstract.py index c7c3fa2..574094c 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -1,7 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import sys import datetime from deprecated import deprecated @@ -27,88 +26,29 @@ from enum import Enum from .exceptions import PyMISPInvalidFormat, PyMISPError +from collections.abc import MutableMapping +from functools import lru_cache +from pathlib import Path + logger = logging.getLogger('pymisp') -if sys.version_info < (3, 0): - from collections import MutableMapping - import os - from cachetools import cached, LRUCache +resources_path = Path(__file__).parent / 'data' +misp_objects_path = resources_path / 'misp-objects' / 'objects' +with (resources_path / 'describeTypes.json').open('r') as f: + describe_types = load(f)['result'] - resources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') - misp_objects_path = os.path.join(resources_path, 'misp-objects', 'objects') - with open(os.path.join(resources_path, 'describeTypes.json'), 'r') as f: - describe_types = load(f)['result'] - # This is required because Python 2 is a pain. - from datetime import tzinfo, timedelta +class MISPFileCache(object): + # cache up to 150 JSON structures in class attribute - class UTC(tzinfo): - """UTC""" - - def utcoffset(self, dt): - return timedelta(0) - - def tzname(self, dt): - return "UTC" - - def dst(self, dt): - return timedelta(0) - - class MISPFileCache(object): - # cache up to 150 JSON structures in class attribute - - @staticmethod - @cached(cache=LRUCache(maxsize=150)) - def _load_json(path): - if not os.path.exists(path): - return None - with open(path, 'r') as f: - data = load(f) - return data - -elif sys.version_info < (3, 4): - from collections.abc import MutableMapping - from functools import lru_cache - import os - - resources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') - misp_objects_path = os.path.join(resources_path, 'misp-objects', 'objects') - with open(os.path.join(resources_path, 'describeTypes.json'), 'r') as f: - describe_types = load(f)['result'] - - class MISPFileCache(object): - # cache up to 150 JSON structures in class attribute - - @staticmethod - @lru_cache(maxsize=150) - def _load_json(path): - if not os.path.exists(path): - return None - with open(path, 'r') as f: - data = load(f) - return data - -else: - from collections.abc import MutableMapping - from functools import lru_cache - from pathlib import Path - - resources_path = Path(__file__).parent / 'data' - misp_objects_path = resources_path / 'misp-objects' / 'objects' - with (resources_path / 'describeTypes.json').open('r') as f: - describe_types = load(f)['result'] - - class MISPFileCache(object): - # cache up to 150 JSON structures in class attribute - - @staticmethod - @lru_cache(maxsize=150) - def _load_json(path): - if not path.exists(): - return None - with path.open('r') as f: - data = load(f) - return data + @staticmethod + @lru_cache(maxsize=150) + def _load_json(path): + if not path.exists(): + return None + with path.open('r') as f: + data = load(f) + return data class Distribution(Enum): @@ -191,7 +131,7 @@ class AbstractMISP(MutableMapping, MISPFileCache): To do so, you need to call the respective add_* or update_* methods in ExpandedPyMISP/PyMISP. """ - super(AbstractMISP, self).__init__() + super().__init__() self.__edited = True # As we create a new object, we assume it is edited self.__not_jsonable = [] self.__self_defined_describe_types = None @@ -230,7 +170,7 @@ class AbstractMISP(MutableMapping, MISPFileCache): @misp_objects_path.setter def misp_objects_path(self, misp_objects_path): - if sys.version_info >= (3, 0) and isinstance(misp_objects_path, str): + if isinstance(misp_objects_path, str): misp_objects_path = Path(misp_objects_path) self.__misp_objects_path = misp_objects_path @@ -362,17 +302,14 @@ class AbstractMISP(MutableMapping, MISPFileCache): # The private members don't matter # If we already have a key with that name, we're modifying it. self.__edited = True - super(AbstractMISP, self).__setattr__(name, value) + super().__setattr__(name, value) def _datetime_to_timestamp(self, d): """Convert a datetime.datetime object to a timestamp (int)""" - if isinstance(d, (int, float, str)) or (sys.version_info < (3, 0) and isinstance(d, unicode)): + if isinstance(d, (int, float, str)): # Assume we already have a timestamp return int(d) - if sys.version_info >= (3, 3): - return int(d.timestamp()) - else: - return int((d - datetime.datetime.fromtimestamp(0, UTC())).total_seconds()) + return int(d.timestamp()) def __add_tag(self, tag=None, **kwargs): """Add a tag to the attribute (by name or a MISPTag object)""" @@ -422,13 +359,10 @@ class MISPTag(AbstractMISP): _fields_for_feed = {'name', 'colour'} - def __init__(self): - super(MISPTag, self).__init__() - def from_dict(self, **kwargs): if kwargs.get('Tag'): kwargs = kwargs.get('Tag') - super(MISPTag, self).from_dict(**kwargs) + super().from_dict(**kwargs) def _set_default(self): if not hasattr(self, 'colour'): @@ -437,4 +371,4 @@ class MISPTag(AbstractMISP): def _to_feed(self): if hasattr(self, 'exportable') and not self.exportable: return False - return super(MISPTag, self)._to_feed() + return super()._to_feed() diff --git a/pymisp/api.py b/pymisp/api.py index fb0f394..137fc8d 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1,77 +1,54 @@ - +#!/usr/bin/env python3 # -*- coding: utf-8 -*- -"""Python API using the REST interface of MISP""" - -import copy -import sys -import json -import datetime -from dateutil.parser import parse -import os -import base64 -import re +from typing import TypeVar, Optional, Tuple, List, Dict, Union +from datetime import date, datetime +import csv +from pathlib import Path import logging -from io import BytesIO, open -import zipfile -from deprecated import deprecated +from urllib.parse import urljoin +import json +import requests +from requests.auth import AuthBase +import re +from uuid import UUID +import warnings +import sys -from . import __version__, warning_2020 -from .exceptions import PyMISPError, SearchError, NoURL, NoKey, PyMISPEmptyResponse -from .mispevent import MISPEvent, MISPAttribute, MISPUser, MISPOrganisation, MISPSighting, MISPFeed, MISPObject, MISPSharingGroup -from .abstract import AbstractMISP, pymisp_json_default, describe_types +from . import __version__, everything_broken +from .exceptions import MISPServerError, PyMISPUnexpectedResponse, PyMISPNotImplementedYet, PyMISPError, NoURL, NoKey +from .mispevent import MISPEvent, MISPAttribute, MISPSighting, MISPLog, MISPObject, \ + MISPUser, MISPOrganisation, MISPShadowAttribute, MISPWarninglist, MISPTaxonomy, \ + MISPGalaxy, MISPNoticelist, MISPObjectReference, MISPObjectTemplate, MISPSharingGroup, \ + MISPRole, MISPServer, MISPFeed, MISPEventDelegation, MISPCommunity, MISPUserSetting +from .abstract import pymisp_json_default, MISPTag, AbstractMISP, describe_types + +SearchType = TypeVar('SearchType', str, int) +# str: string to search / list: values to search (OR) / dict: {'OR': [list], 'NOT': [list], 'AND': [list]} +SearchParameterTypes = TypeVar('SearchParameterTypes', str, List[SearchType], Dict[str, SearchType]) +DateTypes = TypeVar('DateTypes', datetime, date, SearchType, float) +DateInterval = TypeVar('DateInterval', DateTypes, Tuple[DateTypes, DateTypes]) + +ToIDSType = TypeVar('ToIDSType', str, int, bool) logger = logging.getLogger('pymisp') -try: - from urllib.parse import urljoin - # Least dirty way to support python 2 and 3 - basestring = str - unicode = str -except ImportError: - from urlparse import urljoin -try: - import requests - HAVE_REQUESTS = True -except ImportError: - HAVE_REQUESTS = False - -try: - from requests_futures.sessions import FuturesSession - ASYNC_OK = True -except ImportError: - ASYNC_OK = False - -everything_broken = '''Unknown error: the response is not in JSON. -Something is broken server-side, please send us everything that follows (careful with the auth key): -Request headers: -{} -Request body: -{} -Response (if any): -{}''' - - -class PyMISP(object): # pragma: no cover +class PyMISP: """Python API for MISP :param url: URL of the MISP instance you want to connect to :param key: API key of the user you want to use :param ssl: can be True or False (to check ot not the validity of the certificate. Or a CA_BUNDLE in case of self signed certificate (the concatenation of all the *.crt of the chain) - :param out_type: Type of object (json) NOTE: XML output isn't supported anymore, keeping the flag for compatibility reasons. :param debug: Write all the debug information to stderr :param proxies: Proxy dict as describes here: http://docs.python-requests.org/en/master/user/advanced/#proxies :param cert: Client certificate, as described there: http://docs.python-requests.org/en/master/user/advanced/#client-side-certificates - :param asynch: Use asynchronous processing where possible :param auth: The auth parameter is passed directly to requests, as described here: http://docs.python-requests.org/en/master/user/authentication/ :param tool: The software using PyMISP (string), used to set a unique user-agent """ - warning_2020() - - @deprecated(reason="Please use ExpandedPyMISP instead (requires Python 3.6+). This class will be an alias of ExpandedPyMISP early 2020 and your code will most probably fail.", action='default') - def __init__(self, url, key, ssl=True, out_type='json', debug=None, proxies=None, cert=None, asynch=False, auth=None, tool=None): + def __init__(self, url: str, key: str, ssl=True, debug: bool=False, proxies: dict={}, + cert: Tuple[str, tuple]=None, auth: AuthBase=None, tool: str=''): if not url: raise NoURL('Please provide the URL of your MISP instance.') if not key: @@ -82,2026 +59,698 @@ class PyMISP(object): # pragma: no cover self.ssl = ssl self.proxies = proxies self.cert = cert - self.asynch = asynch self.auth = auth self.tool = tool - if asynch and not ASYNC_OK: - logger.critical("You turned on Async, but don't have requests_futures installed") - self.asynch = False - self.resources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') - if out_type != 'json': - raise PyMISPError('The only output type supported by PyMISP is JSON. If you still rely on XML, use PyMISP v2.4.49') + self.global_pythonify = False + + self.resources_path = Path(__file__).parent / 'data' if debug: logger.setLevel(logging.DEBUG) logger.info('To configure logging in your script, leave it to None and use the following: import logging; logging.getLogger(\'pymisp\').setLevel(logging.DEBUG)') try: # Make sure the MISP instance is working and the URL is valid - response = self.get_recommended_api_version() + response = self.recommended_pymisp_version if response.get('errors'): logger.warning(response.get('errors')[0]) - elif not response.get('version'): - logger.warning("Unable to check the recommended PyMISP version (MISP <2.4.60), please upgrade.") else: pymisp_version_tup = tuple(int(x) for x in __version__.split('.')) recommended_version_tup = tuple(int(x) for x in response['version'].split('.')) if recommended_version_tup < pymisp_version_tup[:3]: - logger.info("The version of PyMISP recommended by the MISP instance ({}) is older than the one you're using now ({}). If you have a problem, please upgrade the MISP instance or use an older PyMISP version.".format(response['version'], __version__)) + logger.info(f"The version of PyMISP recommended by the MISP instance (response['version']) is older than the one you're using now ({__version__}). If you have a problem, please upgrade the MISP instance or use an older PyMISP version.") elif pymisp_version_tup[:3] < recommended_version_tup: - logger.warning("The version of PyMISP recommended by the MISP instance ({}) is newer than the one you're using now ({}). Please upgrade PyMISP.".format(response['version'], __version__)) + logger.warning(f"The version of PyMISP recommended by the MISP instance ({response['version']}) is newer than the one you're using now ({__version__}). Please upgrade PyMISP.") + misp_version = self.misp_instance_version + if 'version' in misp_version: + self._misp_version = tuple(int(v) for v in misp_version['version'].split('.')) + + # Get the user information + self._current_user, self._current_role, self._current_user_settings = self.get_user(pythonify=True, expanded=True) except Exception as e: - raise PyMISPError('Unable to connect to MISP ({}). Please make sure the API key and the URL are correct (http/https is required): {}'.format(self.root_url, e)) + raise PyMISPError(f'Unable to connect to MISP ({self.root_url}). Please make sure the API key and the URL are correct (http/https is required): {e}') try: - self.describe_types = self.get_live_describe_types() + self.describe_types = self.describe_types_remote except Exception: - self.describe_types = self.get_local_describe_types() + self.describe_types = self.describe_types_local self.categories = self.describe_types['categories'] self.types = self.describe_types['types'] self.category_type_mapping = self.describe_types['category_type_mappings'] self.sane_default = self.describe_types['sane_defaults'] - def __repr__(self): - return '<{self.__class__.__name__}(url={self.root_url})'.format(self=self) + def remote_acl(self, debug_type: str='findMissingFunctionNames'): + """This should return an empty list, unless the ACL is outdated. + debug_type can only be printAllFunctionNames, findMissingFunctionNames, or printRoleAccess + """ + response = self._prepare_request('GET', f'events/queryACL/{debug_type}') + return self._check_response(response, expect_json=True) - @deprecated(reason="Use ExpandedPyMISP.remote_acl", version='2.4.111', action='default') - def get_live_query_acl(self): - """This should return an empty list, unless the ACL is outdated.""" - response = self._prepare_request('GET', urljoin(self.root_url, 'events/queryACL.json')) - return self._check_response(response) - - @deprecated(reason="Use ExpandedPyMISP.describe_types_local", version='2.4.110', action='default') - def get_local_describe_types(self): + @property + def describe_types_local(self): + '''Returns the content of describe types from the package''' return describe_types - @deprecated(reason="Use ExpandedPyMISP.describe_types_remote", version='2.4.110', action='default') - def get_live_describe_types(self): - response = self._prepare_request('GET', urljoin(self.root_url, 'attributes/describeTypes.json')) - remote_describe_types = self._check_response(response) - if remote_describe_types.get('error'): - for e in remote_describe_types.get('error'): - raise PyMISPError('Failed: {}'.format(e)) - remote_describe_types = remote_describe_types['result'] - if not remote_describe_types.get('sane_defaults'): - raise PyMISPError('The MISP server your are trying to reach is outdated (<2.4.52). Please use PyMISP v2.4.51.1 (pip install -I PyMISP==v2.4.51.1) and/or contact your administrator.') - return remote_describe_types - - def _prepare_request(self, request_type, url, data=None, - background_callback=None, output_type='json'): - if logger.isEnabledFor(logging.DEBUG): - logger.debug('{} - {}'.format(request_type, url)) - if data is not None: - logger.debug(data) - if data is None: - req = requests.Request(request_type, url) - else: - if not isinstance(data, str): - if isinstance(data, dict): - # Remove None values. - data = {k: v for k, v in data.items() if v is not None} - data = json.dumps(data, default=pymisp_json_default) - req = requests.Request(request_type, url, data=data) - if self.asynch and background_callback is not None: - local_session = FuturesSession - else: - local_session = requests.Session - with local_session() as s: - ua_suffix = '' - if self.tool: - ua_suffix = ' - {}'.format(self.tool) - req.auth = self.auth - prepped = s.prepare_request(req) - prepped.headers.update( - {'Authorization': self.key, - 'Accept': 'application/{}'.format(output_type), - 'content-type': 'application/{}'.format(output_type), - 'User-Agent': 'PyMISP {} - Python {}.{}.{}{}'.format(__version__, sys.version_info[0], sys.version_info[1], sys.version_info[2], ua_suffix)}) - if logger.isEnabledFor(logging.DEBUG): - logger.debug(prepped.headers) - settings = s.merge_environment_settings(req.url, proxies=self.proxies or {}, stream=None, verify=self.ssl, cert=self.cert) - if self.asynch and background_callback is not None: - return s.send(prepped, background_callback=background_callback, **settings) - else: - return s.send(prepped, **settings) - - # ##################### - # ### Core helpers #### - # ##################### - - def flatten_error_messages(self, response): - """Dirty dirty method to normalize the error messages between the API calls. - Any response containing the a key 'error' or 'errors' failed at some point, - we make one single list out of it. - """ - messages = [] - if response.get('error'): - if isinstance(response['error'], list): - for e in response['error']: - if isinstance(e, dict): - messages.append(e['error']['value'][0]) - else: - messages.append(e) - else: - messages.append(['error']) - elif response.get('errors'): - if isinstance(response['errors'], dict): - for where, errors in response['errors'].items(): - if isinstance(errors, dict): - for where, msg in errors.items(): - if isinstance(msg, list): - for m in msg: - messages.append('Error in {}: {}'.format(where, m)) - else: - messages.append('Error in {}: {}'.format(where, msg)) - else: - if isinstance(errors, list): - for e in errors: - if not e: - continue - if isinstance(e, basestring): - messages.append(e) - continue - for type_e, msgs in e.items(): - for m in msgs: - messages.append('Error in {}: {}'.format(where, m)) - else: - messages.append('{} ({})'.format(errors, where)) - - return messages - - def _check_response(self, response, lenient_response_type=False): - """Check if the response from the server is not an unexpected error""" - try: - json_response = response.json() - except ValueError: - # If the server didn't return a JSON blob, we've a problem. - if not len(response.text): - raise PyMISPEmptyResponse('The server returned an empty response. \n{}\n{}\n'.format(response.request.headers, response.request.body)) - if lenient_response_type and not response.headers.get('content-type').startswith('application/json;'): - return response.text - else: - raise PyMISPError(everything_broken.format(response.request.headers, response.request.body, response.text)) - - errors = [] - - if response.status_code >= 500: - errors.append('500 exception: {}'.format(json_response)) - logger.critical(everything_broken.format(response.request.headers, response.request.body, json_response)) - - to_return = json_response - if isinstance(to_return, (list, str)): - # FIXME: This case look like a bug. - to_return = {'response': to_return} - else: - if to_return.get('error'): - if not isinstance(to_return['error'], list): - errors.append(to_return['error']) - else: - errors += to_return['error'] - if to_return.get('errors'): - if not isinstance(to_return['errors'], list): - errors.append(to_return['errors']) - else: - errors += to_return['errors'] - - if 400 <= response.status_code < 500: - if not errors and to_return.get('message'): - errors.append(to_return['message']) - else: - errors.append(str(response.status_code)) - errors += self.flatten_error_messages(to_return) - if errors: - to_return['errors'] = errors - if logger.isEnabledFor(logging.DEBUG): - logger.debug(json.dumps(to_return, indent=4)) - return to_return - - def _one_or_more(self, value): - """Returns a list/tuple of one or more items, regardless of input.""" - return value if isinstance(value, (tuple, list)) else (value,) - - def _make_mispevent(self, event): - """Transform a Json MISP event into a MISPEvent""" - if not isinstance(event, MISPEvent): - e = MISPEvent(self.describe_types) - e.load(copy.copy(event)) - else: - e = event - return e - - def _prepare_full_event(self, distribution, threat_level_id, analysis, info, date=None, published=False, orgc_id=None, org_id=None, sharing_group_id=None): - """Initialize a new MISPEvent from scratch""" - misp_event = MISPEvent(self.describe_types) - misp_event.from_dict(info=info, distribution=distribution, threat_level_id=threat_level_id, - analysis=analysis, date=date, orgc_id=orgc_id, org_id=org_id, sharing_group_id=sharing_group_id) - if published: - misp_event.publish() - return misp_event - - def _prepare_full_attribute(self, category, type_value, value, to_ids, comment=None, distribution=None, **kwargs): - """Initialize a new MISPAttribute from scratch""" - misp_attribute = MISPAttribute(self.describe_types) - misp_attribute.from_dict(type=type_value, value=value, category=category, - to_ids=to_ids, comment=comment, distribution=distribution, **kwargs) - return misp_attribute - - def _valid_uuid(self, uuid): - """Test if uuid is valid - Will test against CakeText's RFC 4122, i.e - "the third group must start with a 4, - and the fourth group must start with 8, 9, a or b." - - :param uuid: an uuid - """ - regex = re.compile(r'^[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}\Z', re.I) - match = regex.match(uuid) - return bool(match) - - # ################################################ - # ############### Simple REST API ################ - # ################################################ - - @deprecated(reason="Not used, open an issue if required.", version='2.4.110', action='default') - def test_connection(self): - """Test the auth key""" - response = self.get_version() - if response.get('errors'): - raise PyMISPError(response.get('errors')[0]) - return True - - @deprecated(reason="Use ExpandedPyMISP.search_index", action='default') - def get_index(self, filters=None): - """Return the index. - - Warning, there's a limit on the number of results - """ - url = urljoin(self.root_url, 'events/index') - if filters is None: - response = self._prepare_request('GET', url) - else: - response = self._prepare_request('POST', url, json.dumps(filters)) - return self._check_response(response) - - @deprecated(reason="Use ExpandedPyMISP.get_event", action='default') - def get_event(self, event_id): - """Get an event - - :param event_id: Event id to get - """ - url = urljoin(self.root_url, 'events/{}'.format(event_id)) - response = self._prepare_request('GET', url) - return self._check_response(response) - - @deprecated(reason="Use ExpandedPyMISP.get_object", action='default') - def get_object(self, obj_id): - """Get an object - - :param obj_id: Object id to get - """ - url = urljoin(self.root_url, 'objects/view/{}'.format(obj_id)) - response = self._prepare_request('GET', url) - return self._check_response(response) - - @deprecated(reason="Use ExpandedPyMISP.get_attribute", action='default') - def get_attribute(self, att_id): - """Get an attribute - - :param att_id: Attribute id to get - """ - url = urljoin(self.root_url, 'attributes/view/{}'.format(att_id)) - response = self._prepare_request('GET', url) - return self._check_response(response) - - @deprecated(reason="Use ExpandedPyMISP.add_event", action='default') - def add_event(self, event): - """Add a new event - - :param event: Event as JSON object / string to add - """ - url = urljoin(self.root_url, 'events') - if isinstance(event, MISPEvent): - event = event.to_json() - elif not isinstance(event, basestring): - event = json.dumps(event) - response = self._prepare_request('POST', url, event) - return self._check_response(response) - - @deprecated(reason="Use ExpandedPyMISP.update_attribute", action='default') - def update_attribute(self, attribute_id, attribute): - """Update an attribute - - :param attribute_id: Attribute id/uuid to update - :param attribute: Attribute as JSON object / string to add - """ - url = urljoin(self.root_url, 'attributes/{}'.format(attribute_id)) - if isinstance(attribute, MISPAttribute): - attribute = attribute.to_json() - elif not isinstance(attribute, basestring): - attribute = json.dumps(attribute) - response = self._prepare_request('POST', url, attribute) - return self._check_response(response) - - @deprecated(reason="Use ExpandedPyMISP.update_event", action='default') - def update_event(self, event_id, event): - """Update an event - - :param event_id: Event id to update - :param event: Event as JSON object / string to add - """ - url = urljoin(self.root_url, 'events/{}'.format(event_id)) - if isinstance(event, MISPEvent): - event = event.to_json() - elif not isinstance(event, basestring): - event = json.dumps(event) - response = self._prepare_request('POST', url, event) - return self._check_response(response) - - @deprecated(reason="Use ExpandedPyMISP.delete_event", action='default') - def delete_event(self, event_id): - """Delete an event - - :param event_id: Event id to delete - """ - url = urljoin(self.root_url, 'events/{}'.format(event_id)) - response = self._prepare_request('DELETE', url) - return self._check_response(response) - - @deprecated(reason="Use ExpandedPyMISP.delete_attribute", action='default') - def delete_attribute(self, attribute_id, hard_delete=False): - """Delete an attribute by ID""" - if hard_delete: - url = urljoin(self.root_url, 'attributes/delete/{}/1'.format(attribute_id)) - else: - url = urljoin(self.root_url, 'attributes/delete/{}'.format(attribute_id)) - response = self._prepare_request('POST', url) - return self._check_response(response) - - @deprecated(reason="Use ExpandedPyMISP.push_event_to_ZMQ", action='default') - def pushEventToZMQ(self, event_id): - """Force push an event on ZMQ""" - url = urljoin(self.root_url, 'events/pushEventToZMQ/{}.json'.format(event_id)) - response = self._prepare_request('POST', url) - return self._check_response(response) - - @deprecated(reason="Use ExpandedPyMISP.direct_call", action='default') - def direct_call(self, url, data=None): - '''Very lightweight call that posts a data blob (python dictionary or json string) on the URL''' - url = urljoin(self.root_url, url) - if not data: - response = self._prepare_request('GET', url) - else: - if isinstance(data, dict): - data = json.dumps(data) - response = self._prepare_request('POST', url, data) - return self._check_response(response, lenient_response_type=True) - - # ############################################## - # ############### Event handling ############### - # ############################################## - - @deprecated(reason="Use ExpandedPyMISP.get_event", action='default') - def get(self, eid): - """Get an event by event ID""" - return self.get_event(eid) - - @deprecated(reason="Use ExpandedPyMISP.update_event", action='default') - def update(self, event): - """Update an event by ID""" - e = self._make_mispevent(event) - if e.uuid: - eid = e.uuid - else: - eid = e.id - return self.update_event(eid, e) - - @deprecated(reason="Use ExpandedPyMISP.publish", action='default') - def fast_publish(self, event_id, alert=False): - """Does the same as the publish method, but just try to publish the event - even with one single HTTP GET. - The default is to not send a mail as it is assumed this method is called on update. - """ - if not alert: - url = urljoin(self.root_url, 'events/publish/{}'.format(event_id)) - else: - url = urljoin(self.root_url, 'events/alert/{}'.format(event_id)) - response = self._prepare_request('POST', url) - return self._check_response(response) - - @deprecated(reason="Use ExpandedPyMISP.publish", action='default') - def publish(self, event, alert=True): - """Publish event (with or without alert email) - :param event: pass event or event id (as string or int) to publish - :param alert: set to True by default (send alerting email) if False will not send alert - :return publish status - """ - if isinstance(event, int) or (isinstance(event, basestring) and event.isdigit()): - event_id = event - else: - full_event = self._make_mispevent(event) - if full_event.published: - return {'error': 'Already published'} - event_id = full_event.id - return self.fast_publish(event_id, alert) - - @deprecated(reason="Use ExpandedPyMISP.update_event", action='default') - def change_threat_level(self, event, threat_level_id): - """Change the threat level of an event""" - e = self._make_mispevent(event) - e.threat_level_id = threat_level_id - return self.update(e) - - @deprecated(reason="Use ExpandedPyMISP.update_event", action='default') - def change_analysis_status(self, event, analysis_status): - """Change the analysis status of an event""" - e = self._make_mispevent(event) - e.analysis = analysis_status - return self.update(e) - - @deprecated(reason="Use ExpandedPyMISP.update_event", action='default') - def change_distribution(self, event, distribution): - """Change the distribution of an event""" - e = self._make_mispevent(event) - e.distribution = distribution - return self.update(e) - - @deprecated(reason="Use ExpandedPyMISP.change_sharing_group_on_entity", action='default') - def change_sharing_group(self, event, sharing_group_id): - """Change the sharing group of an event""" - e = self._make_mispevent(event) - e.distribution = 4 # Needs to be 'Sharing group' - if 'SharingGroup' in e: # Delete former SharingGroup information - del e.SharingGroup - e.sharing_group_id = sharing_group_id # Set new sharing group id - return self.update(e) - - @deprecated(reason="Use ExpandedPyMISP.add_event", action='default') - def new_event(self, distribution=None, threat_level_id=None, analysis=None, info=None, date=None, published=False, orgc_id=None, org_id=None, sharing_group_id=None): - """Create and add a new event""" - misp_event = self._prepare_full_event(distribution, threat_level_id, analysis, info, date, published, orgc_id, org_id, sharing_group_id) - return self.add_event(misp_event) - - @deprecated(reason="Use ExpandedPyMISP.tag", version='2.4.111', action='default') - def tag(self, uuid, tag): - """Tag an event or an attribute""" - if not self._valid_uuid(uuid): - raise PyMISPError('Invalid UUID') - url = urljoin(self.root_url, 'tags/attachTagToObject') - to_post = {'uuid': uuid, 'tag': tag} - response = self._prepare_request('POST', url, json.dumps(to_post)) - return self._check_response(response) - - @deprecated(reason="Use ExpandedPyMISP.untag", version='2.4.111', action='default') - def untag(self, uuid, tag): - """Untag an event or an attribute""" - if not self._valid_uuid(uuid): - raise PyMISPError('Invalid UUID') - url = urljoin(self.root_url, 'tags/removeTagFromObject') - to_post = {'uuid': uuid, 'tag': tag} - response = self._prepare_request('POST', url, json.dumps(to_post)) - return self._check_response(response) - - # ##### File attributes ##### - def _send_attributes(self, event, attributes, proposal=False): - """ - Helper to add new attributes to an existing event, identified by an event object or an event id - - - :param event: EventID (int) or Event to alter - :param attributes: One or more attribute to add - :param proposal: True or False based on whether the attributes should be proposed or directly save - :type event: MISPEvent, int - :type attributes: MISPAttribute, list - :type proposal: bool - :return: list of responses - :rtype: list - """ - event_id = self._extract_event_id(event) - responses = [] - if not event_id: - raise PyMISPError("Unable to find the ID of the event to update.") - if not attributes: - return [{'error': 'No attributes.'}] - - # Propals need to be posted in single requests - if proposal: - for a in attributes: - # proposal_add(...) returns a dict - responses.append(self.proposal_add(event_id, a)) - else: - url = urljoin(self.root_url, 'attributes/add/{}'.format(event_id)) - if isinstance(attributes, list): - if all(isinstance(a, AbstractMISP) for a in attributes): - data = attributes - else: - values = [] - for a in attributes: - values.append(a['value']) - attributes[0]['value'] = values - data = attributes[0].to_json() - else: - data = attributes.to_json() - # _prepare_request(...) returns a requests.Response Object - resp = self._prepare_request('POST', url, json.dumps(data, default=pymisp_json_default)) - try: - responses.append(resp.json()) - except Exception: - # The response isn't a json object, appending the text. - responses.append(resp.text) - return responses - - def _extract_event_id(self, event): - """ - Extracts the eventId from a given MISPEvent - - :param event: MISPEvent to extract the id from - :type event: MISPEvent - :return: EventId - :rtype: int - """ - event_id = None - if isinstance(event, MISPEvent): - if hasattr(event, 'id'): - event_id = event.id - elif hasattr(event, 'uuid'): - event_id = event.uuid - elif isinstance(event, int) or (isinstance(event, str) and (event.isdigit() or self._valid_uuid(event))): - event_id = event - else: - if 'Event' in event: - e = event['Event'] - else: - e = event - if 'id' in e: - event_id = e['id'] - elif 'uuid' in e: - event_id = e['uuid'] - return event_id - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_named_attribute(self, event, type_value, value, category=None, to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): - """Add one or more attributes to an existing event""" - attributes = [] - for value in self._one_or_more(value): - attributes.append(self._prepare_full_attribute(category, type_value, value, to_ids, comment, distribution, **kwargs)) - return self._send_attributes(event, attributes, proposal) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_hashes(self, event, category='Artifacts dropped', filename=None, md5=None, sha1=None, sha256=None, ssdeep=None, comment=None, to_ids=True, distribution=None, proposal=False, **kwargs): - """Add hashe(s) to an existing event""" - - attributes = [] - type_value = '{}' - value = '' - if filename: - type_value = 'filename|{}' - value = filename + '|' - if md5: - attributes.append(self._prepare_full_attribute(category, type_value.format('md5'), value + md5, to_ids, comment, distribution)) - if sha1: - attributes.append(self._prepare_full_attribute(category, type_value.format('sha1'), value + sha1, to_ids, comment, distribution)) - if sha256: - attributes.append(self._prepare_full_attribute(category, type_value.format('sha256'), value + sha256, to_ids, comment, distribution)) - if ssdeep: - attributes.append(self._prepare_full_attribute(category, type_value.format('ssdeep'), value + ssdeep, to_ids, comment, distribution)) - - return self._send_attributes(event, attributes, proposal) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def av_detection_link(self, event, link, category='Antivirus detection', to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): - """Add AV detection link(s)""" - return self.add_named_attribute(event, 'link', link, category, to_ids, comment, distribution, proposal, **kwargs) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_detection_name(self, event, name, category='Antivirus detection', to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): - """Add AV detection name(s)""" - return self.add_named_attribute(event, 'text', name, category, to_ids, comment, distribution, proposal, **kwargs) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_filename(self, event, filename, category='Artifacts dropped', to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): - """Add filename(s)""" - return self.add_named_attribute(event, 'filename', filename, category, to_ids, comment, distribution, proposal, **kwargs) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_attachment(self, event, attachment, category='Artifacts dropped', to_ids=False, comment=None, distribution=None, proposal=False, filename=None, **kwargs): - """Add an attachment to the MISP event - - :param event: The event to add an attachment to - :param attachment: Either a file handle or a path to a file - will be uploaded - :param filename: Explicitly defined attachment filename - """ - if isinstance(attachment, basestring) and os.path.isfile(attachment): - # We have a file to open - if filename is None: - filename = os.path.basename(attachment) - with open(attachment, "rb") as f: - fileData = f.read() - elif hasattr(attachment, "read"): - # It's a file handle - we can read it but it has no filename - fileData = attachment.read() - if filename is None: - filename = 'attachment' - elif isinstance(attachment, (tuple, list)): - # tuple/list (filename, pseudofile) - if filename is None: - filename = attachment[0] - if hasattr(attachment[1], "read"): - # Pseudo file - fileData = attachment[1].read() - else: - fileData = attachment[1] - else: - # Plain file content, no filename - if filename is None: - filename = 'attachment' - fileData = attachment - - if not isinstance(fileData, bytes): - fileData = fileData.encode() - - # by now we have a string for the file - # we just need to b64 encode it and send it on its way - # also, just decode it to utf-8 to avoid the b'string' format - encodedData = base64.b64encode(fileData).decode("utf-8") - - # Send it on its way - return self.add_named_attribute(event, 'attachment', filename, category, to_ids, comment, distribution, proposal, data=encodedData, **kwargs) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_regkey(self, event, regkey, rvalue=None, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): - """Add a registry key""" - if rvalue: - type_value = 'regkey|value' - value = '{}|{}'.format(regkey, rvalue) - else: - type_value = 'regkey' - value = regkey - - attributes = [] - attributes.append(self._prepare_full_attribute(category, type_value, value, to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_regkeys(self, event, regkeys_values, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): - """Add a registry keys""" - attributes = [] - - for regkey, rvalue in regkeys_values.items(): - if rvalue is not None: - type_value = 'regkey|value' - value = '{}|{}'.format(regkey, rvalue) - else: - type_value = 'regkey' - value = regkey - - attributes.append(self._prepare_full_attribute(category, type_value, value, to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_pattern(self, event, pattern, in_file=True, in_memory=False, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): - """Add a pattern(s) in file or in memory""" - if not (in_file or in_memory): - raise PyMISPError('Invalid pattern type: please use in_memory=True or in_file=True') - itemtype = 'pattern-in-file' if in_file else 'pattern-in-memory' - return self.add_named_attribute(event, itemtype, pattern, category, to_ids, comment, distribution, proposal, **kwargs) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_pipe(self, event, named_pipe, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): - """Add pipes(s)""" - def scrub(s): - if not s.startswith('\\.\\pipe\\'): - s = '\\.\\pipe\\{}'.format(s) - return s - attributes = list(map(scrub, self._one_or_more(named_pipe))) - return self.add_named_attribute(event, 'named pipe', attributes, category, to_ids, comment, distribution, proposal, **kwargs) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_mutex(self, event, mutex, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): - """Add mutex(es)""" - def scrub(s): - if not s.startswith('\\BaseNamedObjects\\'): - s = '\\BaseNamedObjects\\{}'.format(s) - return s - attributes = list(map(scrub, self._one_or_more(mutex))) - return self.add_named_attribute(event, 'mutex', attributes, category, to_ids, comment, distribution, proposal, **kwargs) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_yara(self, event, yara, category='Payload delivery', to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): - """Add yara rule(es)""" - return self.add_named_attribute(event, 'yara', yara, category, to_ids, comment, distribution, proposal, **kwargs) - - # ##### Network attributes ##### - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_ipdst(self, event, ipdst, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): - """Add destination IP(s)""" - return self.add_named_attribute(event, 'ip-dst', ipdst, category, to_ids, comment, distribution, proposal, **kwargs) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_ipsrc(self, event, ipsrc, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): - """Add source IP(s)""" - return self.add_named_attribute(event, 'ip-src', ipsrc, category, to_ids, comment, distribution, proposal, **kwargs) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_hostname(self, event, hostname, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): - """Add hostname(s)""" - return self.add_named_attribute(event, 'hostname', hostname, category, to_ids, comment, distribution, proposal, **kwargs) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_domain(self, event, domain, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): - """Add domain(s)""" - return self.add_named_attribute(event, 'domain', domain, category, to_ids, comment, distribution, proposal, **kwargs) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_domain_ip(self, event, domain, ip, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): - """Add domain|ip""" - if isinstance(ip, str): - ip = [ip] - composed = list(map(lambda x: '%s|%s' % (domain, x), ip)) - return self.add_named_attribute(event, 'domain|ip', composed, category, to_ids, comment, distribution, proposal, **kwargs) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_domains_ips(self, event, domain_ips, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): - """Add multiple domain|ip""" - composed = list(map(lambda x: '%s|%s' % (x[0], x[1]), domain_ips.items())) - return self.add_named_attribute(event, 'domain|ip', composed, category, to_ids, comment, distribution, proposal, **kwargs) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_url(self, event, url, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): - """Add url(s)""" - return self.add_named_attribute(event, 'url', url, category, to_ids, comment, distribution, proposal, **kwargs) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_useragent(self, event, useragent, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): - """Add user agent(s)""" - return self.add_named_attribute(event, 'user-agent', useragent, category, to_ids, comment, distribution, proposal, **kwargs) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_traffic_pattern(self, event, pattern, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): - """Add pattern(s) in traffic""" - return self.add_named_attribute(event, 'pattern-in-traffic', pattern, category, to_ids, comment, distribution, proposal, **kwargs) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_snort(self, event, snort, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): - """Add SNORT rule(s)""" - return self.add_named_attribute(event, 'snort', snort, category, to_ids, comment, distribution, proposal, **kwargs) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_asn(self, event, asn, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): - """Add network ASN""" - return self.add_named_attribute(event, 'AS', asn, category, to_ids, comment, distribution, proposal, **kwargs) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_net_other(self, event, netother, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): - """Add a free text entry""" - return self.add_named_attribute(event, 'other', netother, category, to_ids, comment, distribution, proposal, **kwargs) - - # ##### Email attributes ##### - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_email_src(self, event, email, category='Payload delivery', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): - """Add a source email""" - return self.add_named_attribute(event, 'email-src', email, category, to_ids, comment, distribution, proposal, **kwargs) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_email_dst(self, event, email, category='Payload delivery', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): - """Add a destination email""" - return self.add_named_attribute(event, 'email-dst', email, category, to_ids, comment, distribution, proposal, **kwargs) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_email_subject(self, event, email, category='Payload delivery', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): - """Add an email subject""" - return self.add_named_attribute(event, 'email-subject', email, category, to_ids, comment, distribution, proposal, **kwargs) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_email_attachment(self, event, email, category='Payload delivery', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): - """Add an email atachment""" - return self.add_named_attribute(event, 'email-attachment', email, category, to_ids, comment, distribution, proposal, **kwargs) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_email_header(self, event, email, category='Payload delivery', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): - """Add an email header""" - return self.add_named_attribute(event, 'email-header', email, category, to_ids, comment, distribution, proposal, **kwargs) - - # ##### Target attributes ##### - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_target_email(self, event, target, category='Targeting data', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): - """Add an target email""" - return self.add_named_attribute(event, 'target-email', target, category, to_ids, comment, distribution, proposal, **kwargs) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_target_user(self, event, target, category='Targeting data', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): - """Add an target user""" - return self.add_named_attribute(event, 'target-user', target, category, to_ids, comment, distribution, proposal, **kwargs) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_target_machine(self, event, target, category='Targeting data', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): - """Add an target machine""" - return self.add_named_attribute(event, 'target-machine', target, category, to_ids, comment, distribution, proposal, **kwargs) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_target_org(self, event, target, category='Targeting data', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): - """Add an target organisation""" - return self.add_named_attribute(event, 'target-org', target, category, to_ids, comment, distribution, proposal, **kwargs) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_target_location(self, event, target, category='Targeting data', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): - """Add an target location""" - return self.add_named_attribute(event, 'target-location', target, category, to_ids, comment, distribution, proposal, **kwargs) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_target_external(self, event, target, category='Targeting data', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): - """Add an target external""" - return self.add_named_attribute(event, 'target-external', target, category, to_ids, comment, distribution, proposal, **kwargs) - - # ##### Attribution attributes ##### - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_threat_actor(self, event, target, category='Attribution', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): - """Add an threat actor""" - return self.add_named_attribute(event, 'threat-actor', target, category, to_ids, comment, distribution, proposal, **kwargs) - - # ##### Internal reference attributes ##### - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_internal_link(self, event, reference, category='Internal reference', to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): - """Add an internal link""" - return self.add_named_attribute(event, 'link', reference, category, to_ids, comment, distribution, proposal, **kwargs) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_internal_comment(self, event, reference, category='Internal reference', to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): - """Add an internal comment""" - return self.add_named_attribute(event, 'comment', reference, category, to_ids, comment, distribution, proposal, **kwargs) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_internal_text(self, event, reference, category='Internal reference', to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): - """Add an internal text""" - return self.add_named_attribute(event, 'text', reference, category, to_ids, comment, distribution, proposal, **kwargs) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_internal_other(self, event, reference, category='Internal reference', to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): - """Add an internal reference (type other)""" - return self.add_named_attribute(event, 'other', reference, category, to_ids, comment, distribution, proposal, **kwargs) - - # ##### Other attributes ##### - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_other_comment(self, event, reference, category='Other', to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): - """Add other comment""" - return self.add_named_attribute(event, 'comment', reference, category, to_ids, comment, distribution, proposal, **kwargs) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_other_counter(self, event, reference, category='Other', to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): - """Add other counter""" - return self.add_named_attribute(event, 'counter', reference, category, to_ids, comment, distribution, proposal, **kwargs) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute", action='default') - def add_other_text(self, event, reference, category='Other', to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): - """Add other text""" - return self.add_named_attribute(event, 'text', reference, category, to_ids, comment, distribution, proposal, **kwargs) - - # ################################################## - # ######### Upload samples through the API ######### - # ################################################## - - def _prepare_upload(self, event_id, distribution, to_ids, category, comment, info, - analysis, threat_level_id, advanced_extraction): - """Helper to prepare a sample to upload""" - to_post = {'request': {}} - - if event_id is not None: - try: - event_id = int(event_id) - except ValueError: - pass - if not isinstance(event_id, int): - # New event - misp_event = self._prepare_full_event(distribution, threat_level_id, analysis, info) - to_post['request']['distribution'] = misp_event.distribution - to_post['request']['info'] = misp_event.info - to_post['request']['analysis'] = misp_event.analysis - to_post['request']['threat_level_id'] = misp_event.threat_level_id - else: - if distribution is not None: - to_post['request']['distribution'] = distribution - - default_values = self.sane_default['malware-sample'] - if to_ids is None or not isinstance(to_ids, bool): - to_ids = bool(int(default_values['to_ids'])) - to_post['request']['to_ids'] = to_ids - - if category is None or category not in self.categories: - category = default_values['default_category'] - to_post['request']['category'] = category - - to_post['request']['comment'] = comment - to_post['request']['advanced'] = 1 if advanced_extraction else 0 - return to_post, event_id - - def _encode_file_to_upload(self, filepath_or_bytes): - """Helper to encode a file to upload""" - if isinstance(filepath_or_bytes, basestring): - if os.path.isfile(filepath_or_bytes): - with open(filepath_or_bytes, 'rb') as f: - binblob = f.read() - else: - binblob = filepath_or_bytes.encode() - else: - binblob = filepath_or_bytes - return base64.b64encode(binblob).decode() - - @deprecated(reason="Use MISPEvent.add_attribute with the expand='binary' key", action='default') - def upload_sample(self, filename, filepath_or_bytes, event_id, distribution=None, - to_ids=True, category=None, comment=None, info=None, - analysis=None, threat_level_id=None, advanced_extraction=False): - """Upload a sample""" - to_post, event_id = self._prepare_upload(event_id, distribution, to_ids, category, - comment, info, analysis, threat_level_id, - advanced_extraction) - to_post['request']['files'] = [{'filename': filename, 'data': self._encode_file_to_upload(filepath_or_bytes)}] - return self._upload_sample(to_post, event_id) - - @deprecated(reason="Use MISPEvent.add_attribute with the expand='binary' key", action='default') - def upload_samplelist(self, filepaths, event_id, distribution=None, - to_ids=True, category=None, comment=None, info=None, - analysis=None, threat_level_id=None, advanced_extraction=False): - """Upload a list of samples""" - to_post, event_id = self._prepare_upload(event_id, distribution, to_ids, category, - comment, info, analysis, threat_level_id, - advanced_extraction) - files = [] - for path in filepaths: - if not os.path.isfile(path): - continue - files.append({'filename': os.path.basename(path), 'data': self._encode_file_to_upload(path)}) - to_post['request']['files'] = files - return self._upload_sample(to_post, event_id) - - def _upload_sample(self, to_post, event_id=None): - """Helper to upload a sample""" - if event_id is None: - url = urljoin(self.root_url, 'events/upload_sample') - else: - url = urljoin(self.root_url, 'events/upload_sample/{}'.format(event_id)) - response = self._prepare_request('POST', url, json.dumps(to_post)) - return self._check_response(response) - - # ############################ - # ######## Proposals ######### - # ############################ - - def __query_proposal(self, path, id, attribute=None): - """Helper to prepare a query to handle proposals""" - url = urljoin(self.root_url, 'shadow_attributes/{}/{}'.format(path, id)) - if path in ['add', 'edit']: - query = {'request': {'ShadowAttribute': attribute}} - response = self._prepare_request('POST', url, json.dumps(query, default=pymisp_json_default)) - elif path == 'view': - response = self._prepare_request('GET', url) - else: # accept or discard - response = self._prepare_request('POST', url) - return self._check_response(response) - - @deprecated(reason="Use ExpandedPyMISP.get_attribute_proposal", version='2.4.111', action='default') - def proposal_view(self, event_id=None, proposal_id=None): - """View a proposal""" - if proposal_id is not None and event_id is not None: - return {'error': 'You can only view an event ID or a proposal ID'} - if event_id is not None: - id = event_id - else: - id = proposal_id - return self.__query_proposal('view', id) - - @deprecated(reason="Use ExpandedPyMISP.add_attribute_proposal", version='2.4.111', action='default') - def proposal_add(self, event_id, attribute): - """Add a proposal""" - return self.__query_proposal('add', event_id, attribute) - - @deprecated(reason="Use ExpandedPyMISP.update_attribute_proposal", version='2.4.111', action='default') - def proposal_edit(self, attribute_id, attribute): - """Edit a proposal""" - return self.__query_proposal('edit', attribute_id, attribute) - - @deprecated(reason="Use ExpandedPyMISP.accept_attribute_proposal", version='2.4.111', action='default') - def proposal_accept(self, proposal_id): - """Accept a proposal""" - return self.__query_proposal('accept', proposal_id) - - @deprecated(reason="Use ExpandedPyMISP.discard_attribute_proposal", version='2.4.111', action='default') - def proposal_discard(self, proposal_id): - """Discard a proposal""" - return self.__query_proposal('discard', proposal_id) - - # ############################## - # ###### Attribute update ###### - # ############################## - - @deprecated(reason="Use ExpandedPyMISP.update_attribute and MISPAttribute", action='default') - def change_toids(self, attribute_uuid, to_ids): - """Change the toids flag""" - if to_ids not in [0, 1]: - raise Exception('to_ids can only be 0 or 1') - query = {"to_ids": to_ids} - return self.__query('edit/{}'.format(attribute_uuid), query, controller='attributes') - - @deprecated(reason="Use ExpandedPyMISP.update_attribute and MISPAttribute", action='default') - def change_comment(self, attribute_uuid, comment): - """Change the comment of attribute""" - query = {"comment": comment} - return self.__query('edit/{}'.format(attribute_uuid), query, controller='attributes') - - @deprecated(reason="Use ExpandedPyMISP.update_attribute and MISPAttribute", action='default') - def change_disable_correlation(self, attribute_uuid, disable_correlation): - """Change the disable_correlation flag""" - possible_values = [0, 1, False, True] - if disable_correlation not in possible_values: - raise Exception('disable_correlation can only be in {}'.format(', '.join(possible_values))) - query = {"disable_correlation": disable_correlation} - return self.__query('edit/{}'.format(attribute_uuid), query, controller='attributes') - - # ############################## - # ###### Attribute update ###### - # ############################## - - @deprecated(reason="Use ExpandedPyMISP.freetext", version='2.4.111', action='default') - def freetext(self, event_id, string, adhereToWarninglists=False, distribution=None, returnMetaAttributes=False): - """Pass a text to the freetext importer""" - query = {"value": string} - wl_params = [False, True, 'soft'] - if adhereToWarninglists not in wl_params: - raise Exception('Invalid parameter, adhereToWarninglists Can only be {}'.format(', '.join(wl_params))) - if adhereToWarninglists: - query['adhereToWarninglists'] = adhereToWarninglists - if distribution is not None: - query['distribution'] = distribution - if returnMetaAttributes: - query['returnMetaAttributes'] = returnMetaAttributes - return self.__query('freeTextImport/{}'.format(event_id), query, controller='events') - - # ############################## - # ######## REST Search ######### - # ############################## - - def __query(self, path, query, controller='events', async_callback=None): - """Helper to prepare a search query""" - if query.get('error') is not None: - return query - if controller not in ['events', 'attributes', 'objects', 'sightings']: - raise ValueError('Invalid controller. Can only be {}'.format(', '.join(['events', 'attributes', 'objects', 'sightings']))) - url = urljoin(self.root_url, '{}/{}'.format(controller, path.lstrip('/'))) - - if ASYNC_OK and async_callback: - response = self._prepare_request('POST', url, json.dumps(query), async_callback) - else: - response = self._prepare_request('POST', url, json.dumps(query)) - return self._check_response(response) - - @deprecated(reason="Use ExpandedPyMISP.search_index", version='2.4.111', action='default') - def search_index(self, published=None, eventid=None, tag=None, datefrom=None, - dateuntil=None, eventinfo=None, threatlevel=None, distribution=None, - analysis=None, attribute=None, org=None, async_callback=None, normalize=False, - timestamp=None, sharinggroup=None): - """Search only at the index level. Use ! infront of value as NOT, default OR - If using async, give a callback that takes 2 args, session and response: - basic usage is - pymisp.search_index(..., async_callback=lambda ses,resp: print(resp.json())) - - :param published: Published (0,1) - :param eventid: Evend ID(s) | str or list - :param tag: Tag(s) | str or list - :param datefrom: First date, in format YYYY-MM-DD - :param dateuntil: Last date, in format YYYY-MM-DD - :param eventinfo: Event info(s) to match | str or list - :param threatlevel: Threat level(s) (1,2,3,4) | str or list - :param distribution: Distribution level(s) (0,1,2,3) | str or list - :param analysis: Analysis level(s) (0,1,2) | str or list - :param org: Organisation(s) | str or list - :param async_callback: Function to call when the request returns (if running async) - :param normalize: Normalize output | True or False - :param timestamp: Interval since last update (in second, or 1d, 1h, ...) - :param sharinggroup: The sharing group value - """ - allowed = {'published': published, 'eventid': eventid, 'tag': tag, 'dateuntil': dateuntil, - 'datefrom': datefrom, 'eventinfo': eventinfo, 'threatlevel': threatlevel, - 'distribution': distribution, 'analysis': analysis, 'attribute': attribute, - 'org': org, 'timestamp': timestamp, 'sharinggroup': sharinggroup} - rule_levels = {'distribution': ["0", "1", "2", "3", "!0", "!1", "!2", "!3"], - 'threatlevel': ["1", "2", "3", "4", "!1", "!2", "!3", "!4"], - 'analysis': ["0", "1", "2", "!0", "!1", "!2"]} - buildup_url = "events/index" - - to_post = {} - for rule in allowed.keys(): - - if allowed.get(rule) is None: - continue - param = allowed[rule] - if isinstance(param, bool): - param = int(param) - if not isinstance(param, list): - param = [param] - # param = [x for x in map(str, param)] - if rule in rule_levels: - if not set(param).issubset(rule_levels[rule]): - raise SearchError('Values in your {} are invalid, has to be in {}'.format(rule, ', '.join(str(x) for x in rule_levels[rule]))) - to_post[rule] = '|'.join(str(x) for x in param) - url = urljoin(self.root_url, buildup_url) - - if self.asynch and async_callback: - response = self._prepare_request('POST', url, json.dumps(to_post), async_callback) - else: - response = self._prepare_request('POST', url, json.dumps(to_post)) - res = self._check_response(response) - if normalize: - to_return = {'response': []} - for elem in res['response']: - tmp = {'Event': elem} - to_return['response'].append(tmp) - res = to_return - return res - - @deprecated(reason="Use ExpandedPyMISP.search", version='2.4.111', action='default') - def search_all(self, value): - """Search a value in the whole database""" - query = {'value': value, 'searchall': 1} - return self.__query('restSearch', query) - - def __prepare_rest_search(self, values, not_values): - """Prepare a search, generate the chain processed by the server - - :param values: Values to search - :param not_values: Values that should not be in the response - """ - to_return = [] - if values is not None: - if isinstance(values, list): - to_return += values - else: - to_return.append(values) - if not_values is not None: - if isinstance(not_values, list): - to_return += ['!{}'.format(v) for v in not_values] - else: - to_return.append('!{}'.format(not_values)) - return to_return - - @deprecated(reason="Use ExpandedPyMISP.search", version='2.4.111', action='default') - def search(self, controller='events', async_callback=None, **kwargs): - """Search via the Rest API - - :param values: values to search for - :param not_values: values *not* to search for - :param type_attribute: Type of attribute - :param category: Category to search - :param org: Org reporting the event - :param tags: Tags to search for - :param not_tags: Tags *not* to search for - :param date_from: First date - :param date_to: Last date - :param last: Last published events (for example 5d or 12h or 30m) - :param eventid: Evend ID(s) | str or list - :param withAttachments: return events with or without the attachments - :param uuid: search by uuid - :param publish_timestamp: the publish timestamp - :param timestamp: the timestamp of the last modification. Can be a list (from->to) - :param enforceWarninglist: Enforce the warning lists - :param includeWarninglistHits: Include the warning list hits - :param searchall: full text search on the database - :param metadata: return only metadata if True - :param published: return only published events - :param to_ids: return only the attributes with the to_ids flag set - :param deleted: also return the deleted attributes - :param event_timestamp: the timestamp of the last modification of the event (attributes controller only)). Can be a list (from->to) - :param includeProposals: return shadow attributes if True - :param async_callback: The function to run when results are returned - """ - query = {} - # Event: array('value', 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'searchall', 'metadata', 'published'); - # Attribute: array('value', 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'to_ids', 'deleted'); - val = self.__prepare_rest_search(kwargs.pop('values', None), kwargs.pop('not_values', None)) - if val: - query['value'] = val - - query['type'] = kwargs.pop('type_attribute', None) - query['category'] = kwargs.pop('category', None) - query['org'] = kwargs.pop('org', None) - - tag = self.__prepare_rest_search(kwargs.pop('tags', None), kwargs.pop('not_tags', None)) - if tag: - query['tags'] = tag - - date_from = kwargs.pop('date_from', None) - if date_from: - if isinstance(date_from, datetime.date) or isinstance(date_from, datetime.datetime): - query['from'] = date_from.strftime('%Y-%m-%d') - else: - query['from'] = date_from - - date_to = kwargs.pop('date_to', None) - if date_to: - if isinstance(date_to, datetime.date) or isinstance(date_to, datetime.datetime): - query['to'] = date_to.strftime('%Y-%m-%d') - else: - query['to'] = date_to - - query['last'] = kwargs.pop('last', None) - query['eventid'] = kwargs.pop('eventid', None) - query['withAttachments'] = kwargs.pop('withAttachments', None) - - uuid = kwargs.pop('uuid', None) - if uuid: - if self._valid_uuid(uuid): - query['uuid'] = uuid - else: - return {'error': 'You must enter a valid uuid.'} - - returnFormat = kwargs.pop('returnFormat', None) - if returnFormat: - if returnFormat in ['json', 'openioc', 'xml', 'suricata', 'snort', 'text', 'rpz', 'csv', 'cache', 'stix', 'stix2']: - query['returnFormat'] = returnFormat - else: - return {'error': 'You must enter a valid returnFormat - json, openioc, xml, suricata, snort, text, rpz, csv, stix, stix2 or cache'} - else: - query['returnFormat'] = 'json' - - query['publish_timestamp'] = kwargs.pop('publish_timestamp', None) - query['timestamp'] = kwargs.pop('timestamp', None) - query['enforceWarninglist'] = kwargs.pop('enforceWarninglist', None) - query['includeWarninglistHits'] = kwargs.pop('includeWarninglistHits', None) - query['to_ids'] = kwargs.pop('to_ids', None) - query['deleted'] = kwargs.pop('deleted', None) - query['published'] = kwargs.pop('published', None) - - if controller == 'events': - # Event search only: - query['searchall'] = kwargs.pop('searchall', None) - query['metadata'] = kwargs.pop('metadata', None) - if controller == 'attributes': - query['event_timestamp'] = kwargs.pop('event_timestamp', None) - query['includeProposals'] = kwargs.pop('includeProposals', None) - - if kwargs: - logger.info('Some unknown parameters are in kwargs. appending as-is: {}'.format(', '.join(kwargs.keys()))) - # Add all other keys as-is. - query.update({k: v for k, v in kwargs.items()}) - - # Cleanup - query = {k: v for k, v in query.items() if v is not None} - - # Create a session, make it async if and only if we have a callback - return self.__query('restSearch', query, controller, async_callback) - - @deprecated(reason="Use ExpandedPyMISP.get_attribute", version='2.4.111', action='default') - def get_attachment(self, attribute_id): - """Get an attachement (not a malware sample) by attribute ID. - Returns the attachment as a bytestream, or a dictionary containing the error message. - - :param attribute_id: Attribute ID to fetched - """ - url = urljoin(self.root_url, 'attributes/download/{}'.format(attribute_id)) - response = self._prepare_request('GET', url) - try: - response.json() - # The query fails, response contains a json blob - return self._check_response(response) - except ValueError: - # content contains the attachment in binary - return response.content - - @deprecated(reason="Use ExpandedPyMISP.search", version='2.4.111', action='default') - def get_yara(self, event_id): - """Get the yara rules from an event""" - url = urljoin(self.root_url, 'attributes/restSearch') - to_post = {'request': {'eventid': event_id, 'type': 'yara'}} - response = self._prepare_request('POST', url, data=json.dumps(to_post)) - result = self._check_response(response) - if result.get('error') is not None: - return False, result.get('error') - if not result.get('response'): - return False, result.get('message') - rules = '\n\n'.join([a['value'] for a in result['response']['Attribute']]) - return True, rules - - @deprecated(reason="Use ExpandedPyMISP.search. Open an issue if needed.", version='2.4.111', action='default') - def download_samples(self, sample_hash=None, event_id=None, all_samples=False, unzip=True): - """Download samples, by hash or event ID. If there are multiple samples in one event, use the all_samples switch - - :param sample_hash: hash of sample - :param event_id: ID of event - :param all_samples: download all samples - :param unzip: whether to unzip or keep zipped - :return: A tuple with (success, [[event_id, sample_hash, sample_as_bytesio], [event_id,...]]) - In case of legacy sample, the sample_hash will be replaced by the zip's filename - """ - url = urljoin(self.root_url, 'attributes/downloadSample') - to_post = {'request': {'hash': sample_hash, 'eventID': event_id, 'allSamples': all_samples}} - response = self._prepare_request('POST', url, data=json.dumps(to_post)) - result = self._check_response(response) - if result.get('error') is not None: - return False, result.get('error') - if not result.get('result'): - return False, result.get('message') - details = [] - for f in result['result']: - decoded = base64.b64decode(f['base64']) - zipped = BytesIO(decoded) - if unzip: - try: - archive = zipfile.ZipFile(zipped) - if f.get('md5') and f['md5'] in archive.namelist(): - # New format - unzipped = BytesIO(archive.open(f['md5'], pwd=b'infected').read()) - details.append([f['event_id'], f['md5'], unzipped]) - else: - # Old format - unzipped = BytesIO(archive.open(f['filename'], pwd=b'infected').read()) - details.append([f['event_id'], f['filename'], unzipped]) - except zipfile.BadZipfile: - # In case the sample isn't zipped - details.append([f['event_id'], f['filename'], zipped]) - else: - details.append([f['event_id'], "{0}.zip".format(f['filename']), zipped]) - return True, details - - @deprecated(reason="Use ExpandedPyMISP.search", version='2.4.111', action='default') - def download_last(self, last): - """Download the last published events. - - :param last: can be defined in days, hours, minutes (for example 5d or 12h or 30m) - """ - return self.search(last=last) - - def _string_to_timestamp(self, date_string): - pydate = parse(date_string) - if sys.version_info >= (3, 3): - # Sane python version - timestamp = pydate.timestamp() - else: - # Whatever - from datetime import timezone # Only for Python < 3.3 - timestamp = (pydate - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds() - return timestamp - - @deprecated(reason="Use ExpandedPyMISP.search", version='2.4.111', action='default') - def get_events_last_modified(self, search_from, search_to=None): - """Download the last modified events. - - :param search_from: Beginning of the interval. Can be either a timestamp, or a date (2000-12-21) - :param search_to: End of the interval. Can be either a timestamp, or a date (2000-12-21) - """ - - search_from = self._string_to_timestamp(search_from) - - if search_to is not None: - search_to = self._string_to_timestamp(search_to) - to_search = [search_from, search_to] - else: - to_search = search_from - - return self.search(timestamp=to_search) - - # ########## Tags ########## - - @deprecated(reason="Use ExpandedPyMISP.tags", version='2.4.111', action='default') - def get_all_tags(self, quiet=False): - """Get all the tags used on the instance""" - url = urljoin(self.root_url, 'tags') - response = self._prepare_request('GET', url) - r = self._check_response(response) - if not quiet or r.get('errors'): - return r - else: - to_return = [] - for tag in r['Tag']: - to_return.append(tag['name']) - return to_return - - @deprecated(reason="Use ExpandedPyMISP.add_tag", version='2.4.111', action='default') - def new_tag(self, name=None, colour="#00ace6", exportable=False, hide_tag=False): - """Create a new tag""" - to_post = {'Tag': {'name': name, 'colour': colour, 'exportable': exportable, 'hide_tag': hide_tag}} - url = urljoin(self.root_url, 'tags/add') - response = self._prepare_request('POST', url, json.dumps(to_post)) - return self._check_response(response) - - # ########## Version ########## - - @deprecated(reason="Use ExpandedPyMISP.version", version='2.4.110', action='default') - def get_api_version(self): - """Returns the current version of PyMISP installed on the system""" + @property + def describe_types_remote(self): + '''Returns the content of describe types from the remote instance''' + response = self._prepare_request('GET', 'attributes/describeTypes.json') + remote_describe_types = self._check_response(response, expect_json=True) + return remote_describe_types['result'] + + @property + def recommended_pymisp_version(self): + """Returns the recommended API version from the server""" + response = self._prepare_request('GET', 'servers/getPyMISPVersion.json') + return self._check_response(response, expect_json=True) + + @property + def version(self): + """Returns the version of PyMISP you're curently using""" return {'version': __version__} - @deprecated(reason="Use ExpandedPyMISP.pymisp_version_master", version='2.4.110', action='default') - def get_api_version_master(self): + @property + def pymisp_version_master(self): """Get the most recent version of PyMISP from github""" r = requests.get('https://raw.githubusercontent.com/MISP/PyMISP/master/pymisp/__init__.py') if r.status_code == 200: version = re.findall("__version__ = '(.*)'", r.text) return {'version': version[0]} - else: - return {'error': 'Impossible to retrieve the version of the master branch.'} + return {'error': 'Impossible to retrieve the version of the master branch.'} - @deprecated(reason="Use ExpandedPyMISP.recommended_pymisp_version", version='2.4.110', action='default') - def get_recommended_api_version(self): - """Returns the recommended API version from the server""" - url = urljoin(self.root_url, 'servers/getPyMISPVersion.json') - response = self._prepare_request('GET', url) - return self._check_response(response) - - @deprecated(reason="Use ExpandedPyMISP.misp_instance_version", version='2.4.110', action='default') - def get_version(self): + @property + def misp_instance_version(self): """Returns the version of the instance.""" - url = urljoin(self.root_url, 'servers/getVersion.json') - response = self._prepare_request('GET', url) - return self._check_response(response) + response = self._prepare_request('GET', 'servers/getVersion.json') + return self._check_response(response, expect_json=True) - @deprecated(reason="Use ExpandedPyMISP.misp_instance_version_master", version='2.4.110', action='default') - def get_version_master(self): + @property + def misp_instance_version_master(self): """Get the most recent version from github""" r = requests.get('https://raw.githubusercontent.com/MISP/MISP/2.4/VERSION.json') if r.status_code == 200: master_version = json.loads(r.text) return {'version': '{}.{}.{}'.format(master_version['major'], master_version['minor'], master_version['hotfix'])} + return {'error': 'Impossible to retrieve the version of the master branch.'} + + def update_misp(self): + response = self._prepare_request('POST', '/servers/update') + return self._check_response(response, expect_json=True) + + def set_server_setting(self, setting: str, value: Union[str, int, bool], force: bool=False): + data = {'value': value, 'force': force} + response = self._prepare_request('POST', f'/servers/serverSettingsEdit/{setting}', data=data) + return self._check_response(response, expect_json=True) + + def get_server_setting(self, setting: str): + response = self._prepare_request('GET', f'/servers/getSetting/{setting}') + return self._check_response(response, expect_json=True) + + def server_settings(self): + response = self._prepare_request('GET', f'/servers/serverSettings') + return self._check_response(response, expect_json=True) + + def restart_workers(self): + response = self._prepare_request('POST', f'/servers/restartWorkers') + return self._check_response(response, expect_json=True) + + def db_schema_diagnostic(self): + response = self._prepare_request('GET', f'/servers/dbSchemaDiagnostic') + return self._check_response(response, expect_json=True) + + def toggle_global_pythonify(self): + self.global_pythonify = not self.global_pythonify + + # ## BEGIN Event ## + + def events(self, pythonify: bool=False): + events = self._prepare_request('GET', 'events') + events = self._check_response(events, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in events: + return events + to_return = [] + for event in events: + e = MISPEvent() + e.from_dict(**event) + to_return.append(e) + return to_return + + def get_event(self, event: Union[MISPEvent, int, str, UUID], deleted: [bool, int, list]=False, pythonify: bool=False): + '''Get an event from a MISP instance''' + event_id = self.__get_uuid_or_id_from_abstract_misp(event) + if deleted: + data = {'deleted': deleted} + event = self._prepare_request('POST', f'events/view/{event_id}', data=data) else: - return {'error': 'Impossible to retrieve the version of the master branch.'} + event = self._prepare_request('GET', f'events/view/{event_id}') + event = self._check_response(event, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in event: + return event + e = MISPEvent() + e.load(event) + return e - # ############## Statistics ################## + def add_event(self, event: MISPEvent, pythonify: bool=False): + '''Add a new event on a MISP instance''' + new_event = self._prepare_request('POST', 'events', data=event) + new_event = self._check_response(new_event, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in new_event: + return new_event + e = MISPEvent() + e.load(new_event) + return e - @deprecated(reason="Use ExpandedPyMISP.attributes_statistics", version='2.4.110', action='default') - def get_attributes_statistics(self, context='type', percentage=None): - """Get attributes statistics from the MISP instance""" - if (context != 'category'): - context = 'type' - if percentage is not None: - url = urljoin(self.root_url, 'attributes/attributeStatistics/{}/{}'.format(context, percentage)) + def update_event(self, event: MISPEvent, event_id: int=None, pythonify: bool=False): + '''Update an event on a MISP instance''' + if event_id is None: + event_id = self.__get_uuid_or_id_from_abstract_misp(event) else: - url = urljoin(self.root_url, 'attributes/attributeStatistics/{}'.format(context)) - response = self._prepare_request('GET', url) - return self._check_response(response) + event_id = self.__get_uuid_or_id_from_abstract_misp(event_id) + updated_event = self._prepare_request('POST', f'events/{event_id}', data=event) + updated_event = self._check_response(updated_event, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in updated_event: + return updated_event + e = MISPEvent() + e.load(updated_event) + return e - @deprecated(reason="Use ExpandedPyMISP.tags_statistics", version='2.4.110', action='default') - def get_tags_statistics(self, percentage=None, name_sort=None): - """Get tags statistics from the MISP instance""" - if percentage is not None: - percentage = 'true' + def delete_event(self, event: Union[MISPEvent, int, str, UUID]): + '''Delete an event from a MISP instance''' + event_id = self.__get_uuid_or_id_from_abstract_misp(event) + response = self._prepare_request('DELETE', f'events/delete/{event_id}') + return self._check_response(response, expect_json=True) + + def publish(self, event: Union[MISPEvent, int, str, UUID], alert: bool=False): + """Publish the event with one single HTTP POST. + The default is to not send a mail as it is assumed this method is called on update. + """ + event_id = self.__get_uuid_or_id_from_abstract_misp(event) + if alert: + response = self._prepare_request('POST', f'events/alert/{event_id}') else: - percentage = 'false' - if name_sort is not None: - name_sort = 'true' + response = self._prepare_request('POST', f'events/publish/{event_id}') + return self._check_response(response, expect_json=True) + + def contact_event_reporter(self, event: Union[MISPEvent, int, str, UUID], message: str): + """Send a message to the reporter of an event""" + event_id = self.__get_uuid_or_id_from_abstract_misp(event) + to_post = {'message': message} + response = self._prepare_request('POST', f'events/contact/{event_id}', data=to_post) + return self._check_response(response, expect_json=True) + + # ## END Event ### + + # ## BEGIN Object ### + + def get_object(self, misp_object: Union[MISPObject, int, str, UUID], pythonify: bool=False): + '''Get an object from the remote MISP instance''' + object_id = self.__get_uuid_or_id_from_abstract_misp(misp_object) + misp_object = self._prepare_request('GET', f'objects/view/{object_id}') + misp_object = self._check_response(misp_object, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in misp_object: + return misp_object + o = MISPObject(misp_object['Object']['name']) + o.from_dict(**misp_object) + return o + + def add_object(self, event: Union[MISPEvent, int, str, UUID], misp_object: MISPObject, pythonify: bool=False): + '''Add a MISP Object to an existing MISP event''' + event_id = self.__get_uuid_or_id_from_abstract_misp(event) + new_object = self._prepare_request('POST', f'objects/add/{event_id}', data=misp_object) + new_object = self._check_response(new_object, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in new_object: + return new_object + o = MISPObject(new_object['Object']['name']) + o.from_dict(**new_object) + return o + + def update_object(self, misp_object: MISPObject, object_id: int=None, pythonify: bool=False): + '''Update an object on a MISP instance''' + if object_id is None: + object_id = self.__get_uuid_or_id_from_abstract_misp(misp_object) else: - name_sort = 'false' - url = urljoin(self.root_url, 'tags/tagStatistics/{}/{}'.format(percentage, name_sort)) - response = self._prepare_request('GET', url) - return self._check_response(response) + object_id = self.__get_uuid_or_id_from_abstract_misp(object_id) + updated_object = self._prepare_request('POST', f'objects/edit/{object_id}', data=misp_object) + updated_object = self._check_response(updated_object, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in updated_object: + return updated_object + o = MISPObject(updated_object['Object']['name']) + o.from_dict(**updated_object) + return o - @deprecated(reason="Use ExpandedPyMISP.users_statistics", version='2.4.110', action='default') - def get_users_statistics(self, context='data'): - """Get users statistics from the MISP instance""" - availables_contexts = ['data', 'orgs', 'users', 'tags', 'attributehistogram', 'sightings', 'attackMatrix'] - if context not in availables_contexts: - context = 'data' - url = urljoin(self.root_url, 'users/statistics/{}.json'.format(context)) - response = self._prepare_request('GET', url) - return self._check_response(response) + def delete_object(self, misp_object: Union[MISPObject, int, str, UUID]): + '''Delete an object from a MISP instance''' + object_id = self.__get_uuid_or_id_from_abstract_misp(misp_object) + response = self._prepare_request('POST', f'objects/delete/{object_id}') + return self._check_response(response, expect_json=True) - # ############## Sightings ################## + def add_object_reference(self, misp_object_reference: MISPObjectReference, pythonify: bool=False): + """Add a reference to an object""" + object_reference = self._prepare_request('POST', 'object_references/add', misp_object_reference) + object_reference = self._check_response(object_reference, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in object_reference: + return object_reference + r = MISPObjectReference() + r.from_dict(**object_reference) + return r - @deprecated(reason="Use ExpandedPyMISP.add_sighting", version='2.4.110', action='default') - def sighting_per_id(self, attribute_id): - """Add a sighting to an attribute (by attribute ID)""" - url = urljoin(self.root_url, 'sightings/add/{}'.format(attribute_id)) - response = self._prepare_request('POST', url) - return self._check_response(response) + def delete_object_reference(self, object_reference: Union[MISPObjectReference, int, str, UUID]): + """Delete a reference to an object""" + object_reference_id = self.__get_uuid_or_id_from_abstract_misp(object_reference) + response = self._prepare_request('POST', f'object_references/delete/{object_reference_id}') + return self._check_response(response, expect_json=True) - @deprecated(reason="Use ExpandedPyMISP.add_sighting", version='2.4.110', action='default') - def sighting_per_uuid(self, attribute_uuid): - """Add a sighting to an attribute (by attribute UUID)""" - url = urljoin(self.root_url, 'sightings/add/{}'.format(attribute_uuid)) - response = self._prepare_request('POST', url) - return self._check_response(response) + # Object templates - @deprecated(reason="Use ExpandedPyMISP.add_sighting", version='2.4.110', action='default') - def set_sightings(self, sightings): - """Push a sighting (python dictionary or MISPSighting) or a list of sightings""" - if not isinstance(sightings, list): - sightings = [sightings] + def object_templates(self, pythonify: bool=False): + """Get all the object templates.""" + object_templates = self._prepare_request('GET', 'objectTemplates') + object_templates = self._check_response(object_templates, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in object_templates: + return object_templates + to_return = [] + for object_template in object_templates: + o = MISPObjectTemplate() + o.from_dict(**object_template) + to_return.append(o) + return to_return + + def get_object_template(self, object_template: Union[MISPObjectTemplate, int, str, UUID], pythonify: bool=False): + """Gets the full object template corresponting the UUID passed as parameter""" + object_template_id = self.__get_uuid_or_id_from_abstract_misp(object_template) + object_template = self._prepare_request('GET', f'objectTemplates/view/{object_template_id}') + object_template = self._check_response(object_template, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in object_template: + return object_template + t = MISPObjectTemplate() + t.from_dict(**object_template) + return t + + def update_object_templates(self): + """Trigger an update of the object templates""" + response = self._prepare_request('POST', 'objectTemplates/update') + return self._check_response(response, expect_json=True) + + # ## END Object ### + + # ## BEGIN Attribute ### + + def attributes(self, pythonify: bool=False): + attributes = self._prepare_request('GET', f'attributes/index') + attributes = self._check_response(attributes, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in attributes: + return attributes + to_return = [] + for attribute in attributes: + a = MISPAttribute() + a.from_dict(**attribute) + to_return.append(a) + return to_return + + def get_attribute(self, attribute: Union[MISPAttribute, int, str, UUID], pythonify: bool=False): + '''Get an attribute from a MISP instance''' + attribute_id = self.__get_uuid_or_id_from_abstract_misp(attribute) + attribute = self._prepare_request('GET', f'attributes/view/{attribute_id}') + attribute = self._check_response(attribute, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in attribute: + return attribute + a = MISPAttribute() + a.from_dict(**attribute) + return a + + def add_attribute(self, event: Union[MISPEvent, int, str, UUID], attribute: MISPAttribute, pythonify: bool=False): + '''Add an attribute to an existing MISP event + NOTE MISP 2.4.113+: you can pass a list of attributes. + In that case, the pythonified response is the following: {'attributes': [MISPAttribute], 'errors': {errors by attributes}}''' + event_id = self.__get_uuid_or_id_from_abstract_misp(event) + new_attribute = self._prepare_request('POST', f'attributes/add/{event_id}', data=attribute) + new_attribute = self._check_response(new_attribute, expect_json=True) + if isinstance(attribute, list): + # Multiple attributes were passed at once, the handling is totally different + if not (self.global_pythonify or pythonify): + return new_attribute + to_return = {'attributes': []} + if 'errors' in new_attribute: + to_return['errors'] = new_attribute['errors'] + + for new_attr in new_attribute['Attribute']: + a = MISPAttribute() + a.from_dict(**new_attr) + to_return['attributes'].append(a) + return to_return + + if ('errors' in new_attribute and new_attribute['errors'][0] == 403 + and new_attribute['errors'][1]['message'] == 'You do not have permission to do that.'): + # At this point, we assume the user tried to add an attribute on an event they don't own + # Re-try with a proposal + return self.add_attribute_proposal(event_id, attribute, pythonify) + if not (self.global_pythonify or pythonify) or 'errors' in new_attribute: + return new_attribute + a = MISPAttribute() + a.from_dict(**new_attribute) + return a + + def update_attribute(self, attribute: MISPAttribute, attribute_id: int=None, pythonify: bool=False): + '''Update an attribute on a MISP instance''' + if attribute_id is None: + attribute_id = self.__get_uuid_or_id_from_abstract_misp(attribute) + else: + attribute_id = self.__get_uuid_or_id_from_abstract_misp(attribute_id) + updated_attribute = self._prepare_request('POST', f'attributes/edit/{attribute_id}', data=attribute) + updated_attribute = self._check_response(updated_attribute, expect_json=True) + if 'errors' in updated_attribute: + if (updated_attribute['errors'][0] == 403 + and updated_attribute['errors'][1]['message'] == 'You do not have permission to do that.'): + # At this point, we assume the user tried to update an attribute on an event they don't own + # Re-try with a proposal + return self.update_attribute_proposal(attribute_id, attribute, pythonify) + if not (self.global_pythonify or pythonify) or 'errors' in updated_attribute: + return updated_attribute + a = MISPAttribute() + a.from_dict(**updated_attribute) + return a + + def delete_attribute(self, attribute: Union[MISPAttribute, int, str, UUID], hard: bool=False): + '''Delete an attribute from a MISP instance''' + attribute_id = self.__get_uuid_or_id_from_abstract_misp(attribute) + data = {} + if hard: + data['hard'] = 1 + response = self._prepare_request('POST', f'attributes/delete/{attribute_id}', data=data) + response = self._check_response(response, expect_json=True) + if ('errors' in response and response['errors'][0] == 403 + and response['errors'][1]['message'] == 'You do not have permission to do that.'): + # FIXME: https://github.com/MISP/MISP/issues/4913 + # At this point, we assume the user tried to delete an attribute on an event they don't own + # Re-try with a proposal + return self.delete_attribute_proposal(attribute_id) + return response + + # ## END Attribute ### + + # ## BEGIN Attribute Proposal ### + + def attribute_proposals(self, event: Union[MISPEvent, int, str, UUID]=None, pythonify: bool=False): + if event: + event_id = self.__get_uuid_or_id_from_abstract_misp(event) + attribute_proposals = self._prepare_request('GET', f'shadow_attributes/index/{event_id}') + else: + attribute_proposals = self._prepare_request('GET', f'shadow_attributes') + attribute_proposals = self._check_response(attribute_proposals, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in attribute_proposals: + return attribute_proposals + to_return = [] + for attribute_proposal in attribute_proposals: + a = MISPShadowAttribute() + a.from_dict(**attribute_proposal) + to_return.append(a) + return to_return + + def get_attribute_proposal(self, proposal: Union[MISPShadowAttribute, int, str, UUID], pythonify: bool=False): + proposal_id = self.__get_uuid_or_id_from_abstract_misp(proposal) + attribute_proposal = self._prepare_request('GET', f'shadow_attributes/view/{proposal_id}') + attribute_proposal = self._check_response(attribute_proposal, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in attribute_proposal: + return attribute_proposal + a = MISPShadowAttribute() + a.from_dict(**attribute_proposal) + return a + + # NOTE: the tree following method have a very specific meaning, look at the comments + + def add_attribute_proposal(self, event: Union[MISPEvent, int, str, UUID], attribute: MISPAttribute, pythonify: bool=False): + '''Propose a new attribute in an event''' + event_id = self.__get_uuid_or_id_from_abstract_misp(event) + new_attribute_proposal = self._prepare_request('POST', f'shadow_attributes/add/{event_id}', data=attribute) + new_attribute_proposal = self._check_response(new_attribute_proposal, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in new_attribute_proposal: + return new_attribute_proposal + a = MISPShadowAttribute() + a.from_dict(**new_attribute_proposal) + return a + + def update_attribute_proposal(self, initial_attribute: Union[MISPAttribute, int, str, UUID], attribute: MISPAttribute, pythonify: bool=False): + '''Propose a change for an attribute''' + initial_attribute_id = self.__get_uuid_or_id_from_abstract_misp(initial_attribute) + update_attribute_proposal = self._prepare_request('POST', f'shadow_attributes/edit/{initial_attribute_id}', data=attribute) + update_attribute_proposal = self._check_response(update_attribute_proposal, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in update_attribute_proposal: + return update_attribute_proposal + a = MISPShadowAttribute() + a.from_dict(**update_attribute_proposal) + return a + + def delete_attribute_proposal(self, attribute: Union[MISPAttribute, int, str, UUID]): + '''Propose the deletion of an attribute''' + attribute_id = self.__get_uuid_or_id_from_abstract_misp(attribute) + response = self._prepare_request('POST', f'shadow_attributes/delete/{attribute_id}') + return self._check_response(response, expect_json=True) + + # NOTE: You cannot modify an existing proposal, only accept/discard + + def accept_attribute_proposal(self, proposal: Union[MISPShadowAttribute, int, str, UUID]): + '''Accept a proposal''' + proposal_id = self.__get_uuid_or_id_from_abstract_misp(proposal) + response = self._prepare_request('POST', f'shadow_attributes/accept/{proposal_id}') + return self._check_response(response, expect_json=True) + + def discard_attribute_proposal(self, proposal: Union[MISPShadowAttribute, int, str, UUID]): + '''Discard a proposal''' + proposal_id = self.__get_uuid_or_id_from_abstract_misp(proposal) + response = self._prepare_request('POST', f'shadow_attributes/discard/{proposal_id}') + return self._check_response(response, expect_json=True) + + # ## END Attribute Proposal ### + + # ## BEGIN Sighting ### + + def sightings(self, misp_entity: AbstractMISP=None, org: Union[MISPOrganisation, int, str, UUID]=None, pythonify: bool=False): + """Get the list of sighting related to a MISPEvent or a MISPAttribute (depending on type of misp_entity)""" + if isinstance(misp_entity, MISPEvent): + context = 'event' + elif isinstance(misp_entity, MISPAttribute): + context = 'attribute' + else: + context = None + if org is not None: + org_id = self.__get_uuid_or_id_from_abstract_misp(org) + else: + org_id = None + + if context is None: + url = 'sightings' + to_post = {} + else: + url = 'sightings/listSightings' + to_post = {'id': misp_entity.id, 'context': context} + if org_id: + to_post['org_id'] = org_id + sightings = self._prepare_request('POST', url, data=to_post) + + sightings = self._check_response(sightings, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in sightings: + return sightings + to_return = [] for sighting in sightings: - if isinstance(sighting, MISPSighting): - to_post = sighting.to_json() - elif isinstance(sighting, dict): - to_post = json.dumps(sighting) - url = urljoin(self.root_url, 'sightings/add/') - response = self._prepare_request('POST', url, to_post) - return self._check_response(response) + s = MISPSighting() + s.from_dict(**sighting) + to_return.append(s) + return to_return - @deprecated(reason="Not used, open an issue if required.", version='2.4.110', action='default') - def sighting_per_json(self, json_file): - """Push a sighting (JSON file)""" - with open(json_file, 'rb') as f: - jdata = json.load(f) - return self.set_sightings(jdata) - - @deprecated(reason="Use ExpandedPyMISP.add_sighting", version='2.4.110', action='default') - def sighting(self, value=None, uuid=None, id=None, source=None, type=None, timestamp=None, **kwargs): - """ Set a single sighting. - :value: Value of the attribute the sighting is related too. Pushing this object - will update the sighting count of each attriutes with thifs value on the instance - :uuid: UUID of the attribute to update - :id: ID of the attribute to update - :source: Source of the sighting - :type: Type of the sighting - :timestamp: Timestamp associated to the sighting - """ + def add_sighting(self, sighting: MISPSighting, attribute: Union[MISPAttribute, int, str, UUID]=None, pythonify: bool=False): + '''Add a new sighting (globally, or to a specific attribute)''' + if attribute: + attribute_id = self.__get_uuid_or_id_from_abstract_misp(attribute) + new_sighting = self._prepare_request('POST', f'sightings/add/{attribute_id}', data=sighting) + else: + # Either the ID/UUID is in the sighting, or we want to add a sighting on all the attributes with the given value + new_sighting = self._prepare_request('POST', f'sightings/add', data=sighting) + new_sighting = self._check_response(new_sighting, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in new_sighting: + return new_sighting s = MISPSighting() - s.from_dict(value=value, uuid=uuid, id=id, source=source, type=type, timestamp=timestamp, **kwargs) - return self.set_sightings(s) + s.from_dict(**new_sighting) + return s - @deprecated(reason="Use ExpandedPyMISP.sightings", version='2.4.110', action='default') - def sighting_list(self, element_id, scope="attribute", org_id=False): - """Get the list of sighting. - :param element_id: could be an event id or attribute id - :type element_id: int - :param scope: could be attribute or event - :return: A json list of sighting corresponding to the search - :rtype: dict + def delete_sighting(self, sighting: Union[MISPSighting, int, str, UUID]): + '''Delete a sighting from a MISP instance''' + sighting_id = self.__get_uuid_or_id_from_abstract_misp(sighting) + response = self._prepare_request('POST', f'sightings/delete/{sighting_id}') + return self._check_response(response, expect_json=True) - :Example: + # ## END Sighting ### - >>> misp.sighting_list(4731) # default search on attribute - [ ... ] - >>> misp.sighting_list(42, event) # return list of sighting for event 42 - [ ... ] - >>> misp.sighting_list(element_id=42, org_id=2, scope=event) # return list of sighting for event 42 filtered with org id 2 - """ - if isinstance(element_id, int) is False: - raise Exception('Invalid parameter, element_id must be a number') - if scope not in ["attribute", "event"]: - raise Exception('scope parameter must be "attribute" or "event"') - if org_id is not False: - if isinstance(org_id, int) is False: - raise Exception('Invalid parameter, org_id must be a number') - else: - org_id = "" - uri = 'sightings/listSightings/{}/{}/{}'.format(element_id, scope, org_id) - url = urljoin(self.root_url, uri) - response = self._prepare_request('POST', url) - return self._check_response(response) + # ## BEGIN Tags ### - @deprecated(reason="Use ExpandedPyMISP.search_sightings", version='2.4.110', action='default') - def search_sightings(self, context='', async_callback=None, **kwargs): - """Search sightings via the REST API - :context: The context of the search, could be attribute, event or False - :param context_id: ID of the attribute or event if context is specified - :param type_sighting: Type of the sighting - :param date_from: From date - :param date_to: To date - :param publish_timestamp: Last published sighting (e.g. 5m, 3h, 7d) - :param org_id: The org_id - :param source: The source of the sighting - :param include_attribute: Should the result include attribute data - :param include_event: Should the result include event data - :param async_callback: The function to run when results are returned - - :Example: - - >>> misp.search_sightings(**{'publish_timestamp': '30d'}) # search sightings for the last 30 days on the instance - [ ... ] - >>> misp.search_sightings('attribute', context_id=6, include_attribute=1) # return list of sighting for attribute 6 along with the attribute itself - [ ... ] - >>> misp.search_sightings('event', **{'context_id': 17, 'include_event': 1, 'org_id': 2}) # return list of sighting for event 17 filtered with org id 2 - """ - if context not in ['', 'attribute', 'event']: - raise Exception('Context parameter must be empty, "attribute" or "event"') - query = {} - # Sighting: array('id', 'type', 'from', 'to', 'last', 'org_id', 'includeAttribute', 'includeEvent'); - query['returnFormat'] = kwargs.pop('returnFormat', 'json') - query['id'] = kwargs.pop('context_id', None) - query['type'] = kwargs.pop('type_sighting', None) - query['from'] = kwargs.pop('date_from', None) - query['to'] = kwargs.pop('date_to', None) - query['last'] = kwargs.pop('publish_timestamp', None) - query['org_id'] = kwargs.pop('org_id', None) - query['source'] = kwargs.pop('source', None) - query['includeAttribute'] = kwargs.pop('include_attribute', None) - query['includeEvent'] = kwargs.pop('include_event', None) - - # Cleanup - query = {k: v for k, v in query.items() if v is not None} - - if kwargs: - raise SearchError('Unused parameter: {}'.format(', '.join(kwargs.keys()))) - - # Create a session, make it async if and only if we have a callback - controller = 'sightings' - return self.__query('restSearch/' + context, query, controller, async_callback) - - # ############## Sharing Groups ################## - - @deprecated(reason="Use ExpandedPyMISP.sharing_groups", version='2.4.110', action='default') - def get_sharing_groups(self): - """Get the existing sharing groups""" - url = urljoin(self.root_url, 'sharing_groups.json') - response = self._prepare_request('GET', url) - return self._check_response(response) - - # ############## Users ################## - - @deprecated(reason="Use ExpandedPyMISP.users", version='2.4.110', action='default') - def get_users_list(self): - return self._rest_list('admin/users') - - @deprecated(reason="Use ExpandedPyMISP.get_user", version='2.4.110', action='default') - def get_user(self, user_id='me'): - return self._rest_view('users', user_id) - - @deprecated(reason="Use ExpandedPyMISP.add_user", version='2.4.110', action='default') - def add_user(self, email, org_id=None, role_id=None, **kwargs): - if isinstance(email, MISPUser): - # Very dirty, allow to call that from ExpandedPyMISP - new_user = email - else: - new_user = MISPUser() - new_user.from_dict(email=email, org_id=org_id, role_id=role_id, **kwargs) - return self._rest_add('admin/users', new_user) - - @deprecated(reason="Not used, open an issue if required.", version='2.4.110', action='default') - def add_user_json(self, json_file): - with open(json_file, 'rb') as f: - jdata = json.load(f) - new_user = MISPUser() - new_user.from_dict(**jdata) - return self._rest_add('admin/users', new_user) - - @deprecated(reason="Not used, open an issue if required.", version='2.4.110', action='default') - def get_user_fields_list(self): - return self._rest_get_parameters('admin/users') - - @deprecated(reason="Use ExpandedPyMISP.update_user", version='2.4.110', action='default') - def edit_user(self, user_id, **kwargs): - edit_user = MISPUser() - edit_user.from_dict(**kwargs) - return self._rest_edit('admin/users', edit_user, user_id) - - @deprecated(reason="Not used, open an issue if required.", version='2.4.110', action='default') - def edit_user_json(self, json_file, user_id): - with open(json_file, 'rb') as f: - jdata = json.load(f) - new_user = MISPUser() - new_user.from_dict(**jdata) - return self._rest_edit('admin/users', new_user, user_id) - - @deprecated(reason="Use ExpandedPyMISP.delete_user", version='2.4.110', action='default') - def delete_user(self, user_id): - return self._rest_delete('admin/users', user_id) - - # ############## Organisations ################## - - @deprecated(reason="Use ExpandedPyMISP.organisations", version='2.4.110', action='default') - def get_organisations_list(self, scope="local"): - scope = scope.lower() - if scope not in ["local", "external", "all"]: - raise ValueError("Authorized fields are 'local','external' or 'all'") - return self._rest_list('organisations/index/scope:{}'.format(scope)) - - @deprecated(reason="Use ExpandedPyMISP.get_organisation", version='2.4.110', action='default') - def get_organisation(self, organisation_id): - return self._rest_view('organisations', organisation_id) - - @deprecated(reason="Use ExpandedPyMISP.add_organisation", version='2.4.110', action='default') - def add_organisation(self, name, **kwargs): - if isinstance(name, MISPOrganisation): - # Very dirty, allow to call that from ExpandedPyMISP - new_org = name - else: - new_org = MISPOrganisation() - new_org.from_dict(name=name, **kwargs) - if 'local' in new_org: - if new_org.get('local') is False: - if 'uuid' not in new_org: - raise PyMISPError('A remote org MUST have a valid uuid') - return self._rest_add('admin/organisations', new_org) - - @deprecated(reason="Not used, open an issue if required.", version='2.4.110', action='default') - def add_organisation_json(self, json_file): - with open(json_file, 'rb') as f: - jdata = json.load(f) - new_org = MISPOrganisation() - new_org.from_dict(**jdata) - return self._rest_add('admin/organisations', new_org) - - @deprecated(reason="Not used, open an issue if required.", version='2.4.110', action='default') - def get_organisation_fields_list(self): - return self._rest_get_parameters('admin/organisations') - - @deprecated(reason="Use ExpandedPyMISP.update_organisation", version='2.4.110', action='default') - def edit_organisation(self, org_id, **kwargs): - edit_org = MISPOrganisation() - edit_org.from_dict(**kwargs) - return self._rest_edit('admin/organisations', edit_org, org_id) - - @deprecated(reason="Not used, open an issue if required.", version='2.4.110', action='default') - def edit_organisation_json(self, json_file, org_id): - with open(json_file, 'rb') as f: - jdata = json.load(f) - edit_org = MISPOrganisation() - edit_org.from_dict(**jdata) - return self._rest_edit('admin/organisations', edit_org, org_id) - - @deprecated(reason="Use ExpandedPyMISP.delete_organisation", version='2.4.110', action='default') - def delete_organisation(self, org_id): - return self._rest_delete('admin/organisations', org_id) - - # ############## Servers ################## - - def _set_server_organisation(self, server, organisation): - if organisation is None: - raise PyMISPError('Need a valid organisation as argument, create it before if needed') - if 'Organisation' in organisation: - organisation = organisation.get('Organisation') - if 'local' not in organisation: - raise PyMISPError('Need a valid organisation as argument. "local" value have not been set in this organisation') - if 'id' not in organisation: - raise PyMISPError('Need a valid organisation as argument. "id" value doesn\'t exist in provided organisation') - - if organisation.get('local'): # Local organisation is '0' and remote organisation is '1'. These values are extracted from web interface of MISP - organisation_type = 0 - else: - organisation_type = 1 - server['organisation_type'] = organisation_type - server['json'] = json.dumps({'id': organisation['id']}) - return server - - def _set_server_parameters(self, url, name, authkey, organisation, internal, - push, pull, self_signed, push_rules, pull_rules, - submitted_cert, submitted_client_cert, delete_cert, - delete_client_cert): - server = {} - self._set_server_organisation(server, organisation) - if url is not None: - server['url'] = url - if name is not None: - server['name'] = name - if authkey is not None: - server['authkey'] = authkey - if internal is not None: - server['internal'] = internal - if push is not None: - server['push'] = push - if pull is not None: - server['pull'] = pull - if self_signed is not None: - server['self_signed'] = self_signed - if push_rules is not None: - server['push_rules'] = push_rules - if pull_rules is not None: - server['pull_rules'] = pull_rules - if submitted_cert is not None: - server['submitted_cert'] = submitted_cert - if submitted_client_cert is not None: - server['submitted_client_cert'] = submitted_client_cert - if delete_cert is not None: - server['delete_cert'] = delete_cert - if delete_client_cert is not None: - server['delete_client_cert'] = delete_client_cert - return server - - @deprecated(reason="Use ExpandedPyMISP.add_server", version='2.4.110', action='default') - def add_server(self, url, name, authkey, organisation, internal=None, push=False, - pull=False, self_signed=False, push_rules="", pull_rules="", - submitted_cert=None, submitted_client_cert=None): - new_server = self._set_server_parameters(url, name, authkey, organisation, internal, - push, pull, self_signed, push_rules, pull_rules, submitted_cert, - submitted_client_cert, None, None) - url = urljoin(self.root_url, 'servers/add') - response = self._prepare_request('POST', url, json.dumps(new_server)) - return self._check_response(response) - - @deprecated(reason="Not used, open an issue if required.", version='2.4.110', action='default') - def add_server_json(self, json_file): - with open(json_file, 'rb') as f: - jdata = json.load(f) - url = urljoin(self.root_url, 'servers/add') - response = self._prepare_request('POST', url, json.dumps(jdata)) - return self._check_response(response) - - @deprecated(reason="Use ExpandedPyMISP.update_server", version='2.4.110', action='default') - def edit_server(self, server_id, url=None, name=None, authkey=None, organisation=None, internal=None, push=False, - pull=False, self_signed=False, push_rules="", pull_rules="", - submitted_cert=None, submitted_client_cert=None, delete_cert=None, delete_client_cert=None): - new_server = self._set_server_parameters(url, name, authkey, organisation, internal, - push, pull, self_signed, push_rules, pull_rules, submitted_cert, - submitted_client_cert, delete_cert, delete_client_cert) - url = urljoin(self.root_url, 'servers/edit/{}'.format(server_id)) - response = self._prepare_request('POST', url, json.dumps(new_server)) - return self._check_response(response) - - @deprecated(reason="Not used, open an issue if required.", version='2.4.110', action='default') - def edit_server_json(self, json_file, server_id): - with open(json_file, 'rb') as f: - jdata = json.load(f) - url = urljoin(self.root_url, 'servers/edit/{}'.format(server_id)) - response = self._prepare_request('POST', url, json.dumps(jdata)) - return self._check_response(response) - - @deprecated(reason="Use ExpandedPyMISP.server_pull", version='2.4.110', action='default') - def server_pull(self, server_id, event_id=None): - url = urljoin(self.root_url, 'servers/pull/{}'.format(server_id)) - if event_id is not None: - url += '/{}'.format(event_id) - response = self._prepare_request('GET', url) - return self._check_response(response) - - @deprecated(reason="Use ExpandedPyMISP.server_push", version='2.4.110', action='default') - def server_push(self, server_id, event_id=None): - url = urljoin(self.root_url, 'servers/push/{}'.format(server_id)) - if event_id is not None: - url += '/{}'.format(event_id) - response = self._prepare_request('GET', url) - return self._check_response(response) - - @deprecated(reason="Use ExpandedPyMISP.servers", version='2.4.110', action='default') - def servers_index(self): - url = urljoin(self.root_url, 'servers/index') - response = self._prepare_request('GET', url) - return self._check_response(response) - - # ############## Roles ################## - - @deprecated(reason="Use ExpandedPyMISP.roles", version='2.4.110', action='default') - def get_roles_list(self): - """Get the list of existing roles""" - url = urljoin(self.root_url, 'roles') - response = self._prepare_request('GET', url) - return self._check_response(response) - - # ############## Tags ################## - - @deprecated(reason="Use ExpandedPyMISP.tags", version='2.4.110', action='default') - def get_tags_list(self): + def tags(self, pythonify: bool=False): """Get the list of existing tags.""" - url = urljoin(self.root_url, 'tags') - response = self._prepare_request('GET', url) - return self._check_response(response)['Tag'] + tags = self._prepare_request('GET', 'tags') + tags = self._check_response(tags, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in tags: + return tags['Tag'] + to_return = [] + for tag in tags['Tag']: + t = MISPTag() + t.from_dict(**tag) + to_return.append(t) + return to_return - @deprecated(reason="Use ExpandedPyMISP.get_tag", version='2.4.110', action='default') - def get_tag(self, tag_id): + def get_tag(self, tag: Union[MISPTag, int, str, UUID], pythonify: bool=False): """Get a tag by id.""" - url = urljoin(self.root_url, 'tags/view/{}'.format(tag_id)) - response = self._prepare_request('GET', url) - return self._check_response(response) + tag_id = self.__get_uuid_or_id_from_abstract_misp(tag) + tag = self._prepare_request('GET', f'tags/view/{tag_id}') + tag = self._check_response(tag, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in tag: + return tag + t = MISPTag() + t.from_dict(**tag) + return t - def _set_tag_parameters(self, name, colour, exportable, hide_tag, org_id, count, user_id, numerical_value, - attribute_count, old_tag): - tag = old_tag - if name is not None: - tag['name'] = name - if colour is not None: - tag['colour'] = colour - if exportable is not None: - tag['exportable'] = exportable - if hide_tag is not None: - tag['hide_tag'] = hide_tag - if org_id is not None: - tag['org_id'] = org_id - if count is not None: - tag['count'] = count - if user_id is not None: - tag['user_id'] = user_id - if numerical_value is not None: - tag['numerical_value'] = numerical_value - if attribute_count is not None: - tag['attribute_count'] = attribute_count + def add_tag(self, tag: MISPTag, pythonify: bool=False): + '''Add a new tag on a MISP instance + Notes: + * The user calling this method needs the Tag Editor permission + * It doesn't add a tag to an event, simply create it on a MISP instance. + ''' + new_tag = self._prepare_request('POST', 'tags/add', data=tag) + new_tag = self._check_response(new_tag, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in new_tag: + return new_tag + t = MISPTag() + t.from_dict(**new_tag) + return t - return {'Tag': tag} + def enable_tag(self, tag: MISPTag, pythonify: bool=False): + """Enable a tag.""" + tag.hide_tag = False + return self.update_tag(tag, pythonify=pythonify) - @deprecated(reason="Use ExpandedPyMISP.update_tag", version='2.4.110', action='default') - def edit_tag(self, tag_id, name=None, colour=None, exportable=None, hide_tag=None, org_id=None, count=None, - user_id=None, numerical_value=None, attribute_count=None): + def disable_tag(self, tag: MISPTag, pythonify: bool=False): + """Disable a tag.""" + tag.hide_tag = True + return self.update_tag(tag, pythonify=pythonify) + + def update_tag(self, tag: MISPTag, tag_id: int=None, pythonify: bool=False): """Edit only the provided parameters of a tag.""" - old_tag = self.get_tag(tag_id) - new_tag = self._set_tag_parameters(name, colour, exportable, hide_tag, org_id, count, user_id, - numerical_value, attribute_count, old_tag) - url = urljoin(self.root_url, 'tags/edit/{}'.format(tag_id)) - response = self._prepare_request('POST', url, json.dumps(new_tag)) - return self._check_response(response) + if tag_id is None: + tag_id = self.__get_uuid_or_id_from_abstract_misp(tag) + else: + tag_id = self.__get_uuid_or_id_from_abstract_misp(tag_id) + updated_tag = self._prepare_request('POST', f'tags/edit/{tag_id}', data=tag) + updated_tag = self._check_response(updated_tag, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in updated_tag: + return updated_tag + t = MISPTag() + t.from_dict(**updated_tag) + return t - @deprecated(reason="Not used, open an issue if required.", version='2.4.110', action='default') - def edit_tag_json(self, json_file, tag_id): - """Edit the tag using a json file.""" - with open(json_file, 'rb') as f: - jdata = json.load(f) - url = urljoin(self.root_url, 'tags/edit/{}'.format(tag_id)) - response = self._prepare_request('POST', url, json.dumps(jdata)) - return self._check_response(response) + def delete_tag(self, tag: Union[MISPTag, int, str, UUID]): + '''Delete an attribute from a MISP instance''' + tag_id = self.__get_uuid_or_id_from_abstract_misp(tag) + response = self._prepare_request('POST', f'tags/delete/{tag_id}') + return self._check_response(response, expect_json=True) - @deprecated(reason="Use ExpandedPyMISP.enable_tag", version='2.4.110', action='default') - def enable_tag(self, tag_id): - """Enable a tag by id.""" - response = self.edit_tag(tag_id, hide_tag=False) - return response + # ## END Tags ### - @deprecated(reason="Use ExpandedPyMISP.disable_tag", version='2.4.110', action='default') - def disable_tag(self, tag_id): - """Disable a tag by id.""" - response = self.edit_tag(tag_id, hide_tag=True) - return response + # ## BEGIN Taxonomies ### - # ############## Taxonomies ################## - - @deprecated(reason="Use ExpandedPyMISP.taxonomies", version='2.4.110', action='default') - def get_taxonomies_list(self): + def taxonomies(self, pythonify: bool=False): """Get all the taxonomies.""" - url = urljoin(self.root_url, 'taxonomies') - response = self._prepare_request('GET', url) - return self._check_response(response) + taxonomies = self._prepare_request('GET', 'taxonomies') + taxonomies = self._check_response(taxonomies, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in taxonomies: + return taxonomies + to_return = [] + for taxonomy in taxonomies: + t = MISPTaxonomy() + t.from_dict(**taxonomy) + to_return.append(t) + return to_return - @deprecated(reason="Use ExpandedPyMISP.get_taxonomy", version='2.4.110', action='default') - def get_taxonomy(self, taxonomy_id): - """Get a taxonomy by id.""" - url = urljoin(self.root_url, 'taxonomies/view/{}'.format(taxonomy_id)) - response = self._prepare_request('GET', url) - return self._check_response(response) + def get_taxonomy(self, taxonomy: Union[MISPTaxonomy, int, str, UUID], pythonify: bool=False): + """Get a taxonomy from a MISP instance.""" + taxonomy_id = self.__get_uuid_or_id_from_abstract_misp(taxonomy) + taxonomy = self._prepare_request('GET', f'taxonomies/view/{taxonomy_id}') + taxonomy = self._check_response(taxonomy, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in taxonomy: + return taxonomy + t = MISPTaxonomy() + t.from_dict(**taxonomy) + return t + + def enable_taxonomy(self, taxonomy: Union[MISPTaxonomy, int, str, UUID]): + """Enable a taxonomy.""" + taxonomy_id = self.__get_uuid_or_id_from_abstract_misp(taxonomy) + response = self._prepare_request('POST', f'taxonomies/enable/{taxonomy_id}') + return self._check_response(response, expect_json=True) + + def disable_taxonomy(self, taxonomy: Union[MISPTaxonomy, int, str, UUID]): + """Disable a taxonomy.""" + taxonomy_id = self.__get_uuid_or_id_from_abstract_misp(taxonomy) + self.disable_taxonomy_tags(taxonomy_id) + response = self._prepare_request('POST', f'taxonomies/disable/{taxonomy_id}') + return self._check_response(response, expect_json=True) + + def disable_taxonomy_tags(self, taxonomy: Union[MISPTaxonomy, int, str, UUID]): + """Disable all the tags of a taxonomy.""" + taxonomy_id = self.__get_uuid_or_id_from_abstract_misp(taxonomy) + response = self._prepare_request('POST', f'taxonomies/disableTag/{taxonomy_id}') + return self._check_response(response, expect_json=True) + + def enable_taxonomy_tags(self, taxonomy: Union[MISPTaxonomy, int, str, UUID]): + """Enable all the tags of a taxonomy. + NOTE: this automatically done when you call enable_taxonomy.""" + taxonomy_id = self.__get_uuid_or_id_from_abstract_misp(taxonomy) + taxonomy = self.get_taxonomy(taxonomy_id) + if not taxonomy['Taxonomy']['enabled']: + raise PyMISPError(f"The taxonomy {taxonomy['Taxonomy']['name']} is not enabled.") + url = urljoin(self.root_url, 'taxonomies/addTag/{}'.format(taxonomy_id)) + response = self._prepare_request('POST', url) + return self._check_response(response, expect_json=True) - @deprecated(reason="Use ExpandedPyMISP.update_taxonomies", version='2.4.110', action='default') def update_taxonomies(self): """Update all the taxonomies.""" - url = urljoin(self.root_url, 'taxonomies/update') - response = self._prepare_request('POST', url) - return self._check_response(response) + response = self._prepare_request('POST', 'taxonomies/update') + return self._check_response(response, expect_json=True) - @deprecated(reason="Use ExpandedPyMISP.enable_taxonomy", version='2.4.110', action='default') - def enable_taxonomy(self, taxonomy_id): - """Enable a taxonomy by id.""" - url = urljoin(self.root_url, 'taxonomies/enable/{}'.format(taxonomy_id)) - response = self._prepare_request('POST', url) - return self._check_response(response) + # ## END Taxonomies ### - @deprecated(reason="Use ExpandedPyMISP.disable_taxonomy", version='2.4.110', action='default') - def disable_taxonomy(self, taxonomy_id): - """Disable a taxonomy by id.""" - self.disable_taxonomy_tags(taxonomy_id) - url = urljoin(self.root_url, 'taxonomies/disable/{}'.format(taxonomy_id)) - response = self._prepare_request('POST', url) - return self._check_response(response) + # ## BEGIN Warninglists ### - @deprecated(reason="Use ExpandedPyMISP.get_taxonomy", version='2.4.110', action='default') - def get_taxonomy_tags_list(self, taxonomy_id): - """Get all the tags of a taxonomy by id.""" - url = urljoin(self.root_url, 'taxonomies/view/{}'.format(taxonomy_id)) - response = self._prepare_request('GET', url) - return self._check_response(response)["entries"] - - @deprecated(reason="Use ExpandedPyMISP.enable_taxonomy_tags", version='2.4.110', action='default') - def enable_taxonomy_tags(self, taxonomy_id): - """Enable all the tags of a taxonomy by id.""" - enabled = self.get_taxonomy(taxonomy_id)['Taxonomy']['enabled'] - if enabled: - url = urljoin(self.root_url, 'taxonomies/addTag/{}'.format(taxonomy_id)) - response = self._prepare_request('POST', url) - return self._check_response(response) - - @deprecated(reason="Use ExpandedPyMISP.disable_taxonomy_tags", version='2.4.110', action='default') - def disable_taxonomy_tags(self, taxonomy_id): - """Disable all the tags of a taxonomy by id.""" - url = urljoin(self.root_url, 'taxonomies/disableTag/{}'.format(taxonomy_id)) - response = self._prepare_request('POST', url) - return self._check_response(response) - - # ############## WarningLists ################## - - @deprecated(reason="Use ExpandedPyMISP.warninglists", version='2.4.110', action='default') - def get_warninglists(self): + def warninglists(self, pythonify: bool=False): """Get all the warninglists.""" - url = urljoin(self.root_url, 'warninglists') - response = self._prepare_request('GET', url) - return self._check_response(response) + warninglists = self._prepare_request('GET', 'warninglists') + warninglists = self._check_response(warninglists, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in warninglists: + return warninglists['Warninglists'] + to_return = [] + for warninglist in warninglists['Warninglists']: + w = MISPWarninglist() + w.from_dict(**warninglist) + to_return.append(w) + return to_return - @deprecated(reason="Use ExpandedPyMISP.get_warninglist", version='2.4.110', action='default') - def get_warninglist(self, warninglist_id): - """Get a warninglist by id.""" - url = urljoin(self.root_url, 'warninglists/view/{}'.format(warninglist_id)) - response = self._prepare_request('GET', url) - return self._check_response(response) + def get_warninglist(self, warninglist: Union[MISPWarninglist, int, str, UUID], pythonify: bool=False): + """Get a warninglist.""" + warninglist_id = self.__get_uuid_or_id_from_abstract_misp(warninglist) + warninglist = self._prepare_request('GET', f'warninglists/view/{warninglist_id}') + warninglist = self._check_response(warninglist, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in warninglist: + return warninglist + w = MISPWarninglist() + w.from_dict(**warninglist) + return w - @deprecated(reason="Use ExpandedPyMISP.update_warninglists", version='2.4.110', action='default') - def update_warninglists(self): - """Update all the warninglists.""" - url = urljoin(self.root_url, 'warninglists/update') - response = self._prepare_request('POST', url) - return self._check_response(response) - - @deprecated(reason="Please ExpandedPyMISP.toggle_warninglist instead.", action='default') - def toggle_warninglist(self, warninglist_id=None, warninglist_name=None, force_enable=None): + def toggle_warninglist(self, warninglist_id: List[int]=None, warninglist_name: List[str]=None, + force_enable: bool=False): '''Toggle (enable/disable) the status of a warninglist by ID. :param warninglist_id: ID of the WarningList - :param force_enable: Force the warning list in the enabled state (does nothing if already enabled) + :param force_enable: Force the warning list in the enabled state (does nothing is already enabled) ''' if warninglist_id is None and warninglist_name is None: - raise Exception('Either warninglist_id or warninglist_name is required.') + raise PyMISPError('Either warninglist_id or warninglist_name is required.') query = {} if warninglist_id is not None: if not isinstance(warninglist_id, list): @@ -2111,494 +760,1454 @@ class PyMISP(object): # pragma: no cover if not isinstance(warninglist_name, list): warninglist_name = [warninglist_name] query['name'] = warninglist_name - if force_enable is not None: + if force_enable: query['enabled'] = force_enable - url = urljoin(self.root_url, 'warninglists/toggleEnable') - response = self._prepare_request('POST', url, json.dumps(query)) - return self._check_response(response) + response = self._prepare_request('POST', 'warninglists/toggleEnable', data=json.dumps(query)) + return self._check_response(response, expect_json=True) - @deprecated(reason="Use ExpandedPyMISP.enable_warninglist", version='2.4.110', action='default') - def enable_warninglist(self, warninglist_id): - """Enable a warninglist by id.""" + def enable_warninglist(self, warninglist: Union[MISPWarninglist, int, str, UUID]): + """Enable a warninglist.""" + warninglist_id = self.__get_uuid_or_id_from_abstract_misp(warninglist) return self.toggle_warninglist(warninglist_id=warninglist_id, force_enable=True) - @deprecated(reason="Use ExpandedPyMISP.disable_warninglist", version='2.4.110', action='default') - def disable_warninglist(self, warninglist_id): - """Disable a warninglist by id.""" + def disable_warninglist(self, warninglist: Union[MISPWarninglist, int, str, UUID]): + """Disable a warninglist.""" + warninglist_id = self.__get_uuid_or_id_from_abstract_misp(warninglist) return self.toggle_warninglist(warninglist_id=warninglist_id, force_enable=False) - @deprecated(reason='Use ExpandedPyMISP.values_in_warninglist', version='2.4.110', action='default') - def check_warninglist(self, value): + def values_in_warninglist(self, value: list): """Check if IOC values are in warninglist""" - url = urljoin(self.root_url, 'warninglists/checkValue') - response = self._prepare_request('POST', url, json.dumps(value)) - return self._check_response(response) + response = self._prepare_request('POST', 'warninglists/checkValue', data=json.dumps(value)) + return self._check_response(response, expect_json=True) - # ############## NoticeLists ################## + def update_warninglists(self): + """Update all the warninglists.""" + response = self._prepare_request('POST', 'warninglists/update') + return self._check_response(response, expect_json=True) - @deprecated(reason="Use ExpandedPyMISP.noticelists", version='2.4.110', action='default') - def get_noticelists(self): + # ## END Warninglists ### + + # ## BEGIN Noticelist ### + + def noticelists(self, pythonify: bool=False): """Get all the noticelists.""" - url = urljoin(self.root_url, 'noticelists') - response = self._prepare_request('GET', url) - return self._check_response(response) + noticelists = self._prepare_request('GET', 'noticelists') + noticelists = self._check_response(noticelists, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in noticelists: + return noticelists + to_return = [] + for noticelist in noticelists: + n = MISPNoticelist() + n.from_dict(**noticelist) + to_return.append(n) + return to_return - @deprecated(reason="Use ExpandedPyMISP.get_noticelist", version='2.4.110', action='default') - def get_noticelist(self, noticelist_id): + def get_noticelist(self, noticelist: Union[MISPNoticelist, int, str, UUID], pythonify: bool=False): """Get a noticelist by id.""" - url = urljoin(self.root_url, 'noticelists/view/{}'.format(noticelist_id)) - response = self._prepare_request('GET', url) - return self._check_response(response) + noticelist_id = self.__get_uuid_or_id_from_abstract_misp(noticelist) + noticelist = self._prepare_request('GET', f'noticelists/view/{noticelist_id}') + noticelist = self._check_response(noticelist, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in noticelist: + return noticelist + n = MISPNoticelist() + n.from_dict(**noticelist) + return n + + def enable_noticelist(self, noticelist: Union[MISPNoticelist, int, str, UUID]): + """Enable a noticelist by id.""" + # FIXME: https://github.com/MISP/MISP/issues/4856 + # response = self._prepare_request('POST', f'noticelists/enable/{noticelist_id}') + noticelist_id = self.__get_uuid_or_id_from_abstract_misp(noticelist) + response = self._prepare_request('POST', f'noticelists/enableNoticelist/{noticelist_id}/true') + return self._check_response(response, expect_json=True) + + def disable_noticelist(self, noticelist: Union[MISPNoticelist, int, str, UUID]): + """Disable a noticelist by id.""" + # FIXME: https://github.com/MISP/MISP/issues/4856 + # response = self._prepare_request('POST', f'noticelists/disable/{noticelist_id}') + noticelist_id = self.__get_uuid_or_id_from_abstract_misp(noticelist) + response = self._prepare_request('POST', f'noticelists/enableNoticelist/{noticelist_id}') + return self._check_response(response, expect_json=True) - @deprecated(reason="Use ExpandedPyMISP.update_noticelists", version='2.4.110', action='default') def update_noticelists(self): """Update all the noticelists.""" - url = urljoin(self.root_url, 'noticelists/update') - response = self._prepare_request('POST', url) - return self._check_response(response) + response = self._prepare_request('POST', 'noticelists/update') + return self._check_response(response, expect_json=True) - @deprecated(reason="Use ExpandedPyMISP.enable_noticelist", version='2.4.110', action='default') - def enable_noticelist(self, noticelist_id): - """Enable a noticelist by id.""" - url = urljoin(self.root_url, 'noticelists/enableNoticelist/{}/true'.format(noticelist_id)) - response = self._prepare_request('POST', url) - return self._check_response(response) + # ## END Noticelist ### - @deprecated(reason="Use ExpandedPyMISP.disable_noticelist", version='2.4.110', action='default') - def disable_noticelist(self, noticelist_id): - """Disable a noticelist by id.""" - url = urljoin(self.root_url, 'noticelists/enableNoticelist/{}'.format(noticelist_id)) - response = self._prepare_request('POST', url) - return self._check_response(response) + # ## BEGIN Galaxy ### - # ############## Galaxies/Clusters ################## - - @deprecated(reason="Use ExpandedPyMISP.galaxies", version='2.4.110', action='default') - def get_galaxies(self): + def galaxies(self, pythonify: bool=False): """Get all the galaxies.""" - url = urljoin(self.root_url, 'galaxies') - response = self._prepare_request('GET', url) - return self._check_response(response) + galaxies = self._prepare_request('GET', 'galaxies') + galaxies = self._check_response(galaxies, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in galaxies: + return galaxies + to_return = [] + for galaxy in galaxies: + g = MISPGalaxy() + g.from_dict(**galaxy) + to_return.append(g) + return to_return - @deprecated(reason="Use ExpandedPyMISP.get_galaxy", version='2.4.110', action='default') - def get_galaxy(self, galaxy_id): + def get_galaxy(self, galaxy: Union[MISPGalaxy, int, str, UUID], pythonify: bool=False): """Get a galaxy by id.""" - url = urljoin(self.root_url, 'galaxies/view/{}'.format(galaxy_id)) - response = self._prepare_request('GET', url) - return self._check_response(response) + galaxy_id = self.__get_uuid_or_id_from_abstract_misp(galaxy) + galaxy = self._prepare_request('GET', f'galaxies/view/{galaxy_id}') + galaxy = self._check_response(galaxy, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in galaxy: + return galaxy + g = MISPGalaxy() + g.from_dict(**galaxy) + return g - @deprecated(reason="Use ExpandedPyMISP.update_galaxies", version='2.4.110', action='default') def update_galaxies(self): """Update all the galaxies.""" - url = urljoin(self.root_url, 'galaxies/update') - response = self._prepare_request('POST', url) - return self._check_response(response) + response = self._prepare_request('POST', 'galaxies/update') + return self._check_response(response, expect_json=True) - # ############################################## - # ############### Non-JSON output ############## - # ############################################## + # ## END Galaxy ### - # ############## Suricata ############## + # ## BEGIN Feed ### - @deprecated(reason="Use ExpandedPyMISP.search", version='2.4.111', action='default') - def download_all_suricata(self): - """Download all suricata rules events.""" - url = urljoin(self.root_url, 'events/nids/suricata/download') - response = self._prepare_request('GET', url, output_type='rules') - return response + def feeds(self, pythonify: bool=False): + """Get the list of existing feeds.""" + feeds = self._prepare_request('GET', 'feeds') + feeds = self._check_response(feeds, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in feeds: + return feeds + to_return = [] + for feed in feeds: + f = MISPFeed() + f.from_dict(**feed) + to_return.append(f) + return to_return - @deprecated(reason="Use ExpandedPyMISP.search", version='2.4.111', action='default') - def download_suricata_rule_event(self, event_id): - """Download one suricata rule event. + def get_feed(self, feed: Union[MISPFeed, int, str, UUID], pythonify: bool=False): + """Get a feed by id.""" + feed_id = self.__get_uuid_or_id_from_abstract_misp(feed) + feed = self._prepare_request('GET', f'feeds/view/{feed_id}') + feed = self._check_response(feed, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in feed: + return feed + f = MISPFeed() + f.from_dict(**feed) + return f - :param event_id: ID of the event to download (same as get) - """ - url = urljoin(self.root_url, 'events/nids/suricata/download/{}'.format(event_id)) - response = self._prepare_request('GET', url, output_type='rules') - return response + def add_feed(self, feed: MISPFeed, pythonify: bool=False): + '''Add a new feed on a MISP instance''' + # FIXME: https://github.com/MISP/MISP/issues/4834 + feed = {'Feed': feed} + new_feed = self._prepare_request('POST', 'feeds/add', data=feed) + new_feed = self._check_response(new_feed, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in new_feed: + return new_feed + f = MISPFeed() + f.from_dict(**new_feed) + return f - # ############## Text ############### + def enable_feed(self, feed: Union[MISPFeed, int, str, UUID], pythonify: bool=False): + '''Enable a feed (fetching it will create event(s)''' + if not isinstance(feed, MISPFeed): + feed_id = self.__get_uuid_or_id_from_abstract_misp(feed) # In case we have a UUID + feed = MISPFeed() + feed.id = feed_id + feed.enabled = True + return self.update_feed(feed=feed, pythonify=pythonify) - @deprecated(reason="Use ExpandedPyMISP.search", version='2.4.111', action='default') - def get_all_attributes_txt(self, type_attr, tags=False, eventId=False, allowNonIDS=False, date_from=False, date_to=False, last=False, enforceWarninglist=False, allowNotPublished=False): - """Get all attributes from a specific type as plain text. Only published and IDS flagged attributes are exported, except if stated otherwise.""" - url = urljoin(self.root_url, 'attributes/text/download/%s/%s/%s/%s/%s/%s/%s/%s/%s' % (type_attr, tags, eventId, allowNonIDS, date_from, date_to, last, enforceWarninglist, allowNotPublished)) - response = self._prepare_request('GET', url, output_type='txt') - return response + def disable_feed(self, feed: Union[MISPFeed, int, str, UUID], pythonify: bool=False): + '''Disable a feed''' + if not isinstance(feed, MISPFeed): + feed_id = self.__get_uuid_or_id_from_abstract_misp(feed) # In case we have a UUID + feed = MISPFeed() + feed.id = feed_id + feed.enabled = False + return self.update_feed(feed=feed, pythonify=pythonify) - # ############## STIX ############## + def enable_feed_cache(self, feed: Union[MISPFeed, int, str, UUID], pythonify: bool=False): + '''Enable the caching of a feed''' + if not isinstance(feed, MISPFeed): + feed_id = self.__get_uuid_or_id_from_abstract_misp(feed) # In case we have a UUID + feed = MISPFeed() + feed.id = feed_id + feed.caching_enabled = True + return self.update_feed(feed=feed, pythonify=pythonify) - @deprecated(reason="Use ExpandedPyMISP.search", version='2.4.111', action='default') - def get_stix_event(self, event_id=None, with_attachments=False, from_date=False, to_date=False, tags=False): - """Get an event/events in STIX format""" - if tags: - if isinstance(tags, list): - tags = "&&".join(tags) - url = urljoin(self.root_url, "events/stix/download/{}/{}/{}/{}/{}".format( - event_id, with_attachments, tags, from_date, to_date)) - logger.debug("Getting STIX event from %s", url) - response = self._prepare_request('GET', url) - return self._check_response(response) + def disable_feed_cache(self, feed: Union[MISPFeed, int, str, UUID], pythonify: bool=False): + '''Disable the caching of a feed''' + if not isinstance(feed, MISPFeed): + feed_id = self.__get_uuid_or_id_from_abstract_misp(feed) # In case we have a UUID + feed = MISPFeed() + feed.id = feed_id + feed.caching_enabled = False + return self.update_feed(feed=feed, pythonify=pythonify) - @deprecated(reason="Use ExpandedPyMISP.search", version='2.4.111', action='default') - def get_stix(self, **kwargs): - return self.get_stix_event(**kwargs) - - @deprecated(reason="Use ExpandedPyMISP.search", version='2.4.111', action='default') - def get_csv(self, eventid=None, attributes=[], object_attributes=[], misp_types=[], context=False, ignore=False, last=None): - """Get MISP values in CSV format - :param eventid: The event ID to query - :param attributes: The column names to export from normal attributes (i.e. uuid, value, type, ...) - :param object_attributes: The column names to export from attributes within objects (i.e. uuid, value, type, ...) - :param misp_types: MISP types to get (i.e. ip-src, hostname, ...) - :param context: Add event level context (event_info,event_member_org,event_source_org,event_distribution,event_threat_level_id,event_analysis,event_date,event_tag) - :param ignore: Returns the attributes even if the event isn't published, or the attribute doesn't have the to_ids flag set - """ - url = urljoin(self.root_url, 'events/csv/download') - to_post = {} - if eventid: - to_post['eventid'] = eventid - if attributes: - to_post['attributes'] = attributes - if object_attributes: - to_post['object_attributes'] = object_attributes - if misp_types: - for t in misp_types: - if t not in self.types: - logger.warning('{} is not a valid type'.format(t)) - to_post['type'] = misp_types - if context: - to_post['includeContext'] = True - if ignore: - to_post['ignore'] = True - if last: - to_post['last'] = last - if to_post: - response = self._prepare_request('POST', url, json.dumps(to_post), output_type='json') + def update_feed(self, feed: MISPFeed, feed_id: int=None, pythonify: bool=False): + '''Update a feed on a MISP instance''' + if feed_id is None: + feed_id = self.__get_uuid_or_id_from_abstract_misp(feed) else: - response = self._prepare_request('POST', url, output_type='json') - return response.text + feed_id = self.__get_uuid_or_id_from_abstract_misp(feed_id) + # FIXME: https://github.com/MISP/MISP/issues/4834 + feed = {'Feed': feed} + updated_feed = self._prepare_request('POST', f'feeds/edit/{feed_id}', data=feed) + updated_feed = self._check_response(updated_feed, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in updated_feed: + return updated_feed + f = MISPFeed() + f.from_dict(**updated_feed) + return f - # ####################################### - # ######## RestResponse generic ######### - # ####################################### + def delete_feed(self, feed: Union[MISPFeed, int, str, UUID]): + '''Delete a feed from a MISP instance''' + feed_id = self.__get_uuid_or_id_from_abstract_misp(feed) + response = self._prepare_request('POST', f'feeds/delete/{feed_id}') + return self._check_response(response, expect_json=True) - def _rest_list(self, urlpath): - url = urljoin(self.root_url, urlpath) - response = self._prepare_request('GET', url) - return self._check_response(response) - - def _rest_get_parameters(self, urlpath): - url = urljoin(self.root_url, '{}/add'.format(urlpath)) - response = self._prepare_request('GET', url) - return self._check_response(response) - - def _rest_view(self, urlpath, rest_id): - url = urljoin(self.root_url, '{}/view/{}'.format(urlpath, rest_id)) - response = self._prepare_request('GET', url) - return self._check_response(response) - - def _rest_add(self, urlpath, obj): - url = urljoin(self.root_url, '{}/add'.format(urlpath)) - response = self._prepare_request('POST', url, obj.to_json()) - return self._check_response(response) - - def _rest_edit(self, urlpath, obj, rest_id): - url = urljoin(self.root_url, '{}/edit/{}'.format(urlpath, rest_id)) - response = self._prepare_request('POST', url, obj.to_json()) - return self._check_response(response) - - def _rest_delete(self, urlpath, rest_id): - url = urljoin(self.root_url, '{}/delete/{}'.format(urlpath, rest_id)) - response = self._prepare_request('POST', url) - return self._check_response(response) - - # ########################### - # ######## Feed ######### - # ########################### - - @deprecated(reason="Use ExpandedPyMISP.feeds instead", action='default') - def get_feeds_list(self): - """Get the content of all the feeds""" - return self._rest_list('feeds') - - @deprecated(reason="Use ExpandedPyMISP.get_feed instead", action='default') - def get_feed(self, feed_id): - """Get the content of a single feed""" - return self._rest_view('feeds', feed_id) - - @deprecated(reason="Use ExpandedPyMISP.add_feed instead", action='default') - def add_feed(self, source_format, url, name, input_source, provider, **kwargs): - """Delete a feed""" - new_feed = MISPFeed() - new_feed.from_dict(source_format=source_format, url=url, name=name, - input_source=input_source, provider=provider) - return self._rest_add('feeds', new_feed) - - @deprecated(reason="Not used, open an issue if required.", version='2.4.110', action='default') - def get_feed_fields_list(self): - return self._rest_get_parameters('feeds') - - @deprecated(reason="Use ExpandedPyMISP.update_feed instead", action='default') - def edit_feed(self, feed_id, **kwargs): - """Delete a feed""" - edit_feed = MISPFeed() - edit_feed.from_dict(**kwargs) - return self._rest_edit('feeds', edit_feed) - - @deprecated(reason="Use ExpandedPyMISP.delete_feed instead", action='default') - def delete_feed(self, feed_id): - """Delete a feed""" - return self._rest_delete('feeds', feed_id) - - @deprecated(reason="Use ExpandedPyMISP.fetch_feed instead", action='default') - def fetch_feed(self, feed_id): + def fetch_feed(self, feed: Union[MISPFeed, int, str, UUID]): """Fetch one single feed""" - url = urljoin(self.root_url, 'feeds/fetchFromFeed/{}'.format(feed_id)) - response = self._prepare_request('GET', url) + feed_id = self.__get_uuid_or_id_from_abstract_misp(feed) + response = self._prepare_request('GET', f'feeds/fetchFromFeed/{feed_id}') return self._check_response(response) - @deprecated(reason="Use ExpandedPyMISP.cache_all_feeds instead", action='default') - def cache_feeds_all(self): + def cache_all_feeds(self): """ Cache all the feeds""" - url = urljoin(self.root_url, 'feeds/cacheFeeds/all') - response = self._prepare_request('GET', url) + response = self._prepare_request('GET', 'feeds/cacheFeeds/all') return self._check_response(response) - @deprecated(reason="Use ExpandedPyMISP.cache_feeds instead", action='default') - def cache_feed(self, feed_id): + def cache_feed(self, feed: Union[MISPFeed, int, str, UUID]): """Cache a specific feed""" - url = urljoin(self.root_url, 'feeds/cacheFeeds/{}'.format(feed_id)) - response = self._prepare_request('GET', url) + feed_id = self.__get_uuid_or_id_from_abstract_misp(feed) + response = self._prepare_request('GET', f'feeds/cacheFeeds/{feed_id}') return self._check_response(response) - @deprecated(reason="Use ExpandedPyMISP.cache_freetext_feeds instead", action='default') - def cache_feeds_freetext(self): + def cache_freetext_feeds(self): """Cache all the freetext feeds""" - url = urljoin(self.root_url, 'feeds/cacheFeeds/freetext') - response = self._prepare_request('GET', url) + response = self._prepare_request('GET', 'feeds/cacheFeeds/freetext') return self._check_response(response) - @deprecated(reason="Use ExpandedPyMISP.cache_misp_feeds instead", action='default') - def cache_feeds_misp(self): + def cache_misp_feeds(self): """Cache all the MISP feeds""" - url = urljoin(self.root_url, 'feeds/cacheFeeds/misp') - response = self._prepare_request('GET', url) + response = self._prepare_request('GET', 'feeds/cacheFeeds/misp') return self._check_response(response) - @deprecated(reason="Use ExpandedPyMISP.compare_feeds instead", action='default') def compare_feeds(self): """Generate the comparison matrix for all the MISP feeds""" - url = urljoin(self.root_url, 'feeds/compareFeeds') - response = self._prepare_request('GET', url) + response = self._prepare_request('GET', 'feeds/compareFeeds') return self._check_response(response) - @deprecated(reason="Use ExpandedPyMISP.get_feed instead", action='default') - def view_feed(self, feed_ids): - """Alias for get_feed""" - return self.get_feed(feed_ids) + # ## END Feed ### - @deprecated(reason="Use ExpandedPyMISP.feeds instead", action='default') - def view_feeds(self): - """Alias for get_feeds_list""" - return self.get_feeds_list() + # ## BEGIN Server ### - # ###################### - # ### Sharing Groups ### - # ###################### + def servers(self, pythonify: bool=False): + """Get the existing servers the MISP instance can synchronise with""" + servers = self._prepare_request('GET', 'servers') + servers = self._check_response(servers, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in servers: + return servers + to_return = [] + for server in servers: + s = MISPServer() + s.from_dict(**server) + to_return.append(s) + return to_return - @deprecated(reason="Use ExpandedPyMISP.add_sharing_group", version='2.4.111', action='default') - def add_sharing_group(self, name, releasability, description, active=True): - """Add a new sharing group, which includes the organisation associated - with the API key and the local server + def get_sync_config(self, pythonify: bool=False): + '''WARNING: This method only works if the user calling it is a sync user''' + server = self._prepare_request('GET', 'servers/createSync') + server = self._check_response(server, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in server: + return server + s = MISPServer() + s.from_dict(**server) + return s - :name: The name of the sharing group to create - :releasability: The releasibility information - :description: The description of the sharing group - :active: Should the sharing group be set to be active? - """ + def import_server(self, server: MISPServer, pythonify: bool=False): + """Import a sync server config received from get_sync_config""" + server = self._prepare_request('POST', f'servers/import', data=server) + server = self._check_response(server, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in server: + return server + s = MISPServer() + s.from_dict(**server) + return s - new_sg = MISPSharingGroup() - new_sg.from_dict(name=name, releasability=releasability, - description=description, active=active) - return self._rest_add('sharing_groups', new_sg) + def add_server(self, server: MISPServer, pythonify: bool=False): + """Add a server to synchronise with. + Note: You probably fant to use ExpandedPyMISP.get_sync_config and ExpandedPyMISP.import_server instead""" + server = self._prepare_request('POST', f'servers/add', data=server) + server = self._check_response(server, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in server: + return server + s = MISPServer() + s.from_dict(**server) + return s - @deprecated(reason="Use ExpandedPyMISP.add_org_to_sharing_group", version='2.4.111', action='default') - def sharing_group_org_add(self, sharing_group, organisation, extend=False): + def update_server(self, server: MISPServer, server_id: int=None, pythonify: bool=False): + '''Update a server to synchronise with''' + if server_id is None: + server_id = self.__get_uuid_or_id_from_abstract_misp(server) + else: + server_id = self.__get_uuid_or_id_from_abstract_misp(server_id) + updated_server = self._prepare_request('POST', f'servers/edit/{server_id}', data=server) + updated_server = self._check_response(updated_server, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in updated_server: + return updated_server + s = MISPServer() + s.from_dict(**updated_server) + return s + + def delete_server(self, server: Union[MISPServer, int, str, UUID]): + '''Delete a sync server''' + server_id = self.__get_uuid_or_id_from_abstract_misp(server) + response = self._prepare_request('POST', f'servers/delete/{server_id}') + return self._check_response(response, expect_json=True) + + def server_pull(self, server: Union[MISPServer, int, str, UUID], event: Union[MISPEvent, int, str, UUID]=None): + '''Initialize a pull from a sync server''' + server_id = self.__get_uuid_or_id_from_abstract_misp(server) + if event: + event_id = self.__get_uuid_or_id_from_abstract_misp(event) + url = f'servers/pull/{server_id}/{event_id}' + else: + url = f'servers/pull/{server_id}' + response = self._prepare_request('GET', url) + # FIXME: can we pythonify? + return self._check_response(response) + + def server_push(self, server: Union[MISPServer, int, str, UUID], event: Union[MISPEvent, int, str, UUID]=None): + '''Initialize a push to a sync server''' + server_id = self.__get_uuid_or_id_from_abstract_misp(server) + if event: + event_id = self.__get_uuid_or_id_from_abstract_misp(event) + url = f'servers/push/{server_id}/{event_id}' + else: + url = f'servers/push/{server_id}' + response = self._prepare_request('GET', url) + # FIXME: can we pythonify? + return self._check_response(response) + + def test_server(self, server: Union[MISPServer, int, str, UUID]): + server_id = self.__get_uuid_or_id_from_abstract_misp(server) + response = self._prepare_request('POST', f'servers/testConnection/{server_id}') + return self._check_response(response, expect_json=True) + + # ## END Server ### + + # ## BEGIN Sharing group ### + + def sharing_groups(self, pythonify: bool=False): + """Get the existing sharing groups""" + sharing_groups = self._prepare_request('GET', 'sharing_groups') + sharing_groups = self._check_response(sharing_groups, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in sharing_groups: + return sharing_groups + to_return = [] + for sharing_group in sharing_groups: + s = MISPSharingGroup() + s.from_dict(**sharing_group) + to_return.append(s) + return to_return + + def add_sharing_group(self, sharing_group: MISPSharingGroup, pythonify: bool=False): + """Add a new sharing group""" + sharing_group = self._prepare_request('POST', f'sharing_groups/add', data=sharing_group) + sharing_group = self._check_response(sharing_group, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in sharing_group: + return sharing_group + s = MISPSharingGroup() + s.from_dict(**sharing_group) + return s + + def delete_sharing_group(self, sharing_group: Union[MISPSharingGroup, int, str, UUID]): + """Delete a sharing group""" + sharing_group_id = self.__get_uuid_or_id_from_abstract_misp(sharing_group) + response = self._prepare_request('POST', f'sharing_groups/delete/{sharing_group_id}') + return self._check_response(response, expect_json=True) + + def add_org_to_sharing_group(self, sharing_group: Union[MISPSharingGroup, int, str, UUID], + organisation: Union[MISPOrganisation, int, str, UUID], extend: bool=False): '''Add an organisation to a sharing group. :sharing_group: Sharing group's local instance ID, or Sharing group's global UUID :organisation: Organisation's local instance ID, or Organisation's global UUID, or Organisation's name as known to the curent instance :extend: Allow the organisation to extend the group ''' - to_jsonify = {'sg_id': sharing_group, 'org_id': organisation, 'extend': extend} - url = urljoin(self.root_url, 'sharingGroups/addOrg') - response = self._prepare_request('POST', url, json.dumps(to_jsonify)) + sharing_group_id = self.__get_uuid_or_id_from_abstract_misp(sharing_group) + organisation_id = self.__get_uuid_or_id_from_abstract_misp(organisation) + to_jsonify = {'sg_id': sharing_group_id, 'org_id': organisation_id, 'extend': extend} + response = self._prepare_request('POST', 'sharingGroups/addOrg', data=to_jsonify) return self._check_response(response) - @deprecated(reason="Use ExpandedPyMISP.remove_org_from_sharing_group", version='2.4.111', action='default') - def sharing_group_org_remove(self, sharing_group, organisation): + def remove_org_from_sharing_group(self, sharing_group: Union[MISPSharingGroup, int, str, UUID], + organisation: Union[MISPOrganisation, int, str, UUID]): '''Remove an organisation from a sharing group. :sharing_group: Sharing group's local instance ID, or Sharing group's global UUID :organisation: Organisation's local instance ID, or Organisation's global UUID, or Organisation's name as known to the curent instance ''' - to_jsonify = {'sg_id': sharing_group, 'org_id': organisation} - url = urljoin(self.root_url, 'sharingGroups/removeOrg') - response = self._prepare_request('POST', url, json.dumps(to_jsonify)) + sharing_group_id = self.__get_uuid_or_id_from_abstract_misp(sharing_group) + organisation_id = self.__get_uuid_or_id_from_abstract_misp(organisation) + to_jsonify = {'sg_id': sharing_group_id, 'org_id': organisation_id} + response = self._prepare_request('POST', 'sharingGroups/removeOrg', data=to_jsonify) return self._check_response(response) - @deprecated(reason="Use ExpandedPyMISP.add_server_to_sharing_group", version='2.4.111', action='default') - def sharing_group_server_add(self, sharing_group, server, all_orgs=False): + def add_server_to_sharing_group(self, sharing_group: Union[MISPSharingGroup, int, str, UUID], + server: Union[MISPServer, int, str, UUID], all_orgs: bool=False): '''Add a server to a sharing group. :sharing_group: Sharing group's local instance ID, or Sharing group's global UUID :server: Server's local instance ID, or URL of the Server, or Server's name as known to the curent instance :all_orgs: Add all the organisations of the server to the group ''' - to_jsonify = {'sg_id': sharing_group, 'server_id': server, 'all_orgs': all_orgs} - url = urljoin(self.root_url, 'sharingGroups/addServer') - response = self._prepare_request('POST', url, json.dumps(to_jsonify)) + sharing_group_id = self.__get_uuid_or_id_from_abstract_misp(sharing_group) + server_id = self.__get_uuid_or_id_from_abstract_misp(server) + to_jsonify = {'sg_id': sharing_group_id, 'server_id': server_id, 'all_orgs': all_orgs} + response = self._prepare_request('POST', 'sharingGroups/addServer', data=to_jsonify) return self._check_response(response) - @deprecated(reason="Use ExpandedPyMISP.remove_server_from_sharing_group", version='2.4.111', action='default') - def sharing_group_server_remove(self, sharing_group, server): + def remove_server_from_sharing_group(self, sharing_group: Union[MISPSharingGroup, int, str, UUID], + server: Union[MISPServer, int, str, UUID]): '''Remove a server from a sharing group. :sharing_group: Sharing group's local instance ID, or Sharing group's global UUID :server: Server's local instance ID, or URL of the Server, or Server's name as known to the curent instance ''' - to_jsonify = {'sg_id': sharing_group, 'server_id': server} - url = urljoin(self.root_url, 'sharingGroups/removeServer') - response = self._prepare_request('POST', url, json.dumps(to_jsonify)) + sharing_group_id = self.__get_uuid_or_id_from_abstract_misp(sharing_group) + server_id = self.__get_uuid_or_id_from_abstract_misp(server) + to_jsonify = {'sg_id': sharing_group_id, 'server_id': server_id} + response = self._prepare_request('POST', 'sharingGroups/removeServer', data=to_jsonify) return self._check_response(response) - @deprecated(reason="Use ExpandedPyMISP.delete_sharing_group", version='2.4.111', action='default') - def delete_sharing_group(self, sharing_group): - """Delete a sharing group - :sharing_group: Sharing group's local instance ID, or Sharing group's global uuid - """ - return self._rest_delete("sharing_groups", sharing_group) + # ## END Sharing groups ### - # ################### - # ### Objects ### - # ################### + # ## BEGIN Organisation ### - @deprecated(reason="Use ExpandedPyMISP.add_object", version='2.4.111', action='default') - def add_object(self, event_id, *args, **kwargs): - """Add an object - :param event_id: Event ID of the event to attach the object to - :param template_id: Template ID of the template related to that event (not required) - :param misp_object: MISPObject to attach - """ - # NOTE: this slightly fucked up thing is due to the fact template_id was required, and was the 2nd parameter. - template_id = kwargs.get('template_id') - misp_object = kwargs.get('misp_object') - if args: - if isinstance(args[0], MISPObject): - misp_object = args[0] + def organisations(self, scope="local", pythonify: bool=False): + """Get all the organisations.""" + organisations = self._prepare_request('GET', f'organisations/index/scope:{scope}') + organisations = self._check_response(organisations, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in organisations: + return organisations + to_return = [] + for organisation in organisations: + o = MISPOrganisation() + o.from_dict(**organisation) + to_return.append(o) + return to_return + + def get_organisation(self, organisation: Union[MISPOrganisation, int, str, UUID], pythonify: bool=False): + '''Get an organisation.''' + organisation_id = self.__get_uuid_or_id_from_abstract_misp(organisation) + organisation = self._prepare_request('GET', f'organisations/view/{organisation_id}') + organisation = self._check_response(organisation, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in organisation: + return organisation + o = MISPOrganisation() + o.from_dict(**organisation) + return o + + def add_organisation(self, organisation: MISPOrganisation, pythonify: bool=False): + '''Add an organisation''' + new_organisation = self._prepare_request('POST', f'admin/organisations/add', data=organisation) + new_organisation = self._check_response(new_organisation, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in new_organisation: + return new_organisation + o = MISPOrganisation() + o.from_dict(**new_organisation) + return o + + def update_organisation(self, organisation: MISPOrganisation, organisation_id: int=None, pythonify: bool=False): + '''Update an organisation''' + if organisation_id is None: + organisation_id = self.__get_uuid_or_id_from_abstract_misp(organisation) + else: + organisation_id = self.__get_uuid_or_id_from_abstract_misp(organisation_id) + updated_organisation = self._prepare_request('POST', f'admin/organisations/edit/{organisation_id}', data=organisation) + updated_organisation = self._check_response(updated_organisation, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in updated_organisation: + return updated_organisation + o = MISPOrganisation() + o.from_dict(**organisation) + return o + + def delete_organisation(self, organisation: Union[MISPOrganisation, int, str, UUID]): + '''Delete an organisation''' + # NOTE: MISP in inconsistent and currently require "delete" in the path and doesn't support HTTP DELETE + organisation_id = self.__get_uuid_or_id_from_abstract_misp(organisation) + response = self._prepare_request('POST', f'admin/organisations/delete/{organisation_id}') + return self._check_response(response, expect_json=True) + + # ## END Organisation ### + + # ## BEGIN User ### + + def users(self, pythonify: bool=False): + """Get all the users.""" + users = self._prepare_request('GET', 'admin/users') + users = self._check_response(users, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in users: + return users + to_return = [] + for user in users: + u = MISPUser() + u.from_dict(**user) + to_return.append(u) + return to_return + + def get_user(self, user: Union[MISPUser, int, str, UUID]='me', pythonify: bool=False, expanded: bool=False): + '''Get a user. `me` means the owner of the API key doing the query. + expanded also returns a MISPRole and a MISPUserSetting''' + user_id = self.__get_uuid_or_id_from_abstract_misp(user) + user = self._prepare_request('GET', f'users/view/{user_id}') + user = self._check_response(user, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in user: + return user + u = MISPUser() + u.from_dict(**user) + if not expanded: + return u + else: + r = MISPRole() + r.from_dict(**user['Role']) + usersettings = [] + if user['UserSetting']: + for name, value in user['UserSetting'].items(): + us = MISPUserSetting() + us.from_dict(**{'name': name, 'value': value}) + usersettings.append(us) + return u, r, usersettings + + def add_user(self, user: MISPUser, pythonify: bool=False): + '''Add a new user''' + user = self._prepare_request('POST', f'admin/users/add', data=user) + user = self._check_response(user, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in user: + return user + u = MISPUser() + u.from_dict(**user) + return u + + def update_user(self, user: MISPUser, user_id: int=None, pythonify: bool=False): + '''Update an event on a MISP instance''' + if user_id is None: + user_id = self.__get_uuid_or_id_from_abstract_misp(user) + else: + user_id = self.__get_uuid_or_id_from_abstract_misp(user_id) + url = f'users/edit/{user_id}' + if self._current_role.perm_admin or self._current_role.perm_site_admin: + url = f'admin/{url}' + updated_user = self._prepare_request('POST', url, data=user) + updated_user = self._check_response(updated_user, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in updated_user: + return updated_user + e = MISPUser() + e.from_dict(**updated_user) + return e + + def delete_user(self, user: Union[MISPUser, int, str, UUID]): + '''Delete a user''' + # NOTE: MISP in inconsistent and currently require "delete" in the path and doesn't support HTTP DELETE + user_id = self.__get_uuid_or_id_from_abstract_misp(user) + response = self._prepare_request('POST', f'admin/users/delete/{user_id}') + return self._check_response(response, expect_json=True) + + def change_user_password(self, new_password: str, user: Union[MISPUser, int, str, UUID]=None): + response = self._prepare_request('POST', f'users/change_pw', data={'password': new_password}) + return self._check_response(response, expect_json=True) + + # ## END User ### + + # ## BEGIN Role ### + + def roles(self, pythonify: bool=False): + """Get the existing roles""" + roles = self._prepare_request('GET', 'roles') + roles = self._check_response(roles, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in roles: + return roles + to_return = [] + for role in roles: + r = MISPRole() + r.from_dict(**role) + to_return.append(r) + return to_return + + def set_default_role(self, role: Union[MISPRole, int, str, UUID]): + role_id = self.__get_uuid_or_id_from_abstract_misp(role) + url = urljoin(self.root_url, f'/admin/roles/set_default/{role_id}') + response = self._prepare_request('POST', url) + return self._check_response(response, expect_json=True) + + # ## END Role ### + + # ## BEGIN Search methods ### + + def search(self, controller: str='events', return_format: str='json', + limit: Optional[int]=None, page: Optional[int]=None, + value: Optional[SearchParameterTypes]=None, + type_attribute: Optional[SearchParameterTypes]=None, + category: Optional[SearchParameterTypes]=None, + org: Optional[SearchParameterTypes]=None, + tags: Optional[SearchParameterTypes]=None, + quick_filter: Optional[str]=None, quickFilter: Optional[str]=None, + date_from: Optional[DateTypes]=None, + date_to: Optional[DateTypes]=None, + eventid: Optional[SearchType]=None, + with_attachments: Optional[bool]=None, withAttachments: Optional[bool]=None, + metadata: Optional[bool]=None, + uuid: Optional[str]=None, + publish_timestamp: Optional[DateInterval]=None, last: Optional[DateInterval]=None, + timestamp: Optional[DateInterval]=None, + published: Optional[bool]=None, + enforce_warninglist: Optional[bool]=None, enforceWarninglist: Optional[bool]=None, + to_ids: Optional[Union[ToIDSType, List[ToIDSType]]]=None, + deleted: Optional[str]=None, + include_event_uuid: Optional[bool]=None, includeEventUuid: Optional[bool]=None, + include_event_tags: Optional[bool]=None, includeEventTags: Optional[bool]=None, + event_timestamp: Optional[DateTypes]=None, + sg_reference_only: Optional[bool]=None, + eventinfo: Optional[str]=None, + searchall: Optional[bool]=None, + requested_attributes: Optional[str]=None, + include_context: Optional[bool]=None, includeContext: Optional[bool]=None, + headerless: Optional[bool]=None, + include_sightings: Optional[bool]=None, includeSightings: Optional[bool]=None, + include_correlations: Optional[bool]=None, includeCorrelations: Optional[bool]=None, + pythonify: Optional[bool]=False, + **kwargs): + '''Search in the MISP instance + + :param return_format: Set the return format of the search (Currently supported: json, xml, openioc, suricata, snort - more formats are being moved to restSearch with the goal being that all searches happen through this API). Can be passed as the first parameter after restSearch or via the JSON payload. + :param limit: Limit the number of results returned, depending on the scope (for example 10 attributes or 10 full events). + :param page: If a limit is set, sets the page to be returned. page 3, limit 100 will return records 201->300). + :param value: Search for the given value in the attributes' value field. + :param type_attribute: The attribute type, any valid MISP attribute type is accepted. + :param category: The attribute category, any valid MISP attribute category is accepted. + :param org: Search by the creator organisation by supplying the organisation identifier. + :param tags: Tags to search or to exclude. You can pass a list, or the output of `build_complex_query` + :param quick_filter: The string passed to this field will ignore all of the other arguments. MISP will return an xml / json (depending on the header sent) of all events that have a sub-string match on value in the event info, event orgc, or any of the attribute value1 / value2 fields, or in the attribute comment. + :param date_from: Events with the date set to a date after the one specified. This filter will use the date of the event. + :param date_to: Events with the date set to a date before the one specified. This filter will use the date of the event. + :param eventid: The events that should be included / excluded from the search + :param with_attachments: If set, encodes the attachments / zipped malware samples as base64 in the data field within each attribute + :param metadata: Only the metadata (event, tags, relations) is returned, attributes and proposals are omitted. + :param uuid: Restrict the results by uuid. + :param publish_timestamp: Restrict the results by the last publish timestamp (newer than). + :param timestamp: Restrict the results by the timestamp (last edit). Any event with a timestamp newer than the given timestamp will be returned. In case you are dealing with /attributes as scope, the attribute's timestamp will be used for the lookup. + :param published: Set whether published or unpublished events should be returned. Do not set the parameter if you want both. + :param enforce_warninglist: Remove any attributes from the result that would cause a hit on a warninglist entry. + :param to_ids: By default all attributes are returned that match the other filter parameters, irregardless of their to_ids setting. To restrict the returned data set to to_ids only attributes set this parameter to 1. 0 for the ones with to_ids set to False. + :param deleted: If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using "only" as a parameter it will limit the returned data set to soft-deleted data only. + :param include_event_uuid: Instead of just including the event ID, also include the event UUID in each of the attributes. + :param include_event_tags: Include the event level tags in each of the attributes. + :param event_timestamp: Only return attributes from events that have received a modification after the given timestamp. + :param sg_reference_only: If this flag is set, sharing group objects will not be included, instead only the sharing group ID is set. + :param eventinfo: Filter on the event's info field. + :param searchall: Search for a full or a substring (delimited by % for substrings) in the event info, event tags, attribute tags, attribute values or attribute comment fields. + :param requested_attributes: [CSV only] Select the fields that you wish to include in the CSV export. By setting event level fields additionally, includeContext is not required to get event metadata. + :param include_context: [Attribute only] Include the event data with each attribute. + :param headerless: [CSV Only] The CSV created when this setting is set to true will not contain the header row. + :param include_sightings: [JSON Only - Attribute] Include the sightings of the matching attributes. + :param include_correlations: [JSON Only - attribute] Include the correlations of the matching attributes. + :param pythonify: Returns a list of PyMISP Objects instead of the plain json output. Warning: it might use a lot of RAM + + Deprecated: + + :param quickFilter: synponym for quick_filter + :param withAttachments: synonym for with_attachments + :param last: synonym for publish_timestamp + :param enforceWarninglist: synonym for enforce_warninglist + :param includeEventUuid: synonym for include_event_uuid + :param includeEventTags: synonym for include_event_tags + :param includeContext: synonym for include_context + + ''' + + return_formats = ['openioc', 'json', 'xml', 'suricata', 'snort', 'text', 'rpz', 'csv', 'cache', 'stix', 'stix2', 'yara', 'yara-json', 'attack', 'attack-sightings'] + + if controller not in ['events', 'attributes', 'objects', 'sightings']: + raise ValueError('controller has to be in {}'.format(', '.join(['events', 'attributes', 'objects']))) + + # Deprecated stuff / synonyms + if quickFilter is not None: + quick_filter = quickFilter + if withAttachments is not None: + with_attachments = withAttachments + if last is not None: + publish_timestamp = last + if enforceWarninglist is not None: + enforce_warninglist = enforceWarninglist + if includeEventUuid is not None: + include_event_uuid = includeEventUuid + if includeEventTags is not None: + include_event_tags = includeEventTags + if includeContext is not None: + include_context = includeContext + if includeCorrelations is not None: + include_correlations = includeCorrelations + if includeSightings is not None: + include_sightings = includeSightings + # Add all the parameters in kwargs are aimed at modules, or other 3rd party components, and cannot be sanitized. + # They are passed as-is. + query = kwargs + + if return_format not in return_formats: + raise ValueError('return_format has to be in {}'.format(', '.join(return_formats))) + query['returnFormat'] = return_format + + query['page'] = page + query['limit'] = limit + query['value'] = value + query['type'] = type_attribute + query['category'] = category + query['org'] = org + query['tags'] = tags + query['quickFilter'] = quick_filter + query['from'] = self._make_timestamp(date_from) + query['to'] = self._make_timestamp(date_to) + query['eventid'] = eventid + query['withAttachments'] = self._make_misp_bool(with_attachments) + query['metadata'] = self._make_misp_bool(metadata) + query['uuid'] = uuid + if publish_timestamp is not None: + if isinstance(publish_timestamp, (list, tuple)): + query['publish_timestamp'] = (self._make_timestamp(publish_timestamp[0]), self._make_timestamp(publish_timestamp[1])) else: - template_id = args[0] - misp_object = args[1] - - if template_id is not None: - url = urljoin(self.root_url, 'objects/add/{}/{}'.format(event_id, template_id)) + query['publish_timestamp'] = self._make_timestamp(publish_timestamp) + if timestamp is not None: + if isinstance(timestamp, (list, tuple)): + query['timestamp'] = (self._make_timestamp(timestamp[0]), self._make_timestamp(timestamp[1])) + else: + query['timestamp'] = self._make_timestamp(timestamp) + query['published'] = published + query['enforceWarninglist'] = self._make_misp_bool(enforce_warninglist) + if to_ids is not None: + if int(to_ids) not in [0, 1]: + raise ValueError('to_ids has to be in {}'.format(', '.join([0, 1]))) + query['to_ids'] = to_ids + query['deleted'] = deleted + query['includeEventUuid'] = self._make_misp_bool(include_event_uuid) + query['includeEventTags'] = self._make_misp_bool(include_event_tags) + if event_timestamp is not None: + if isinstance(event_timestamp, (list, tuple)): + query['event_timestamp'] = (self._make_timestamp(event_timestamp[0]), self._make_timestamp(event_timestamp[1])) + else: + query['event_timestamp'] = self._make_timestamp(event_timestamp) + query['sgReferenceOnly'] = self._make_misp_bool(sg_reference_only) + query['eventinfo'] = eventinfo + query['searchall'] = searchall + query['requested_attributes'] = requested_attributes + query['includeContext'] = self._make_misp_bool(include_context) + query['headerless'] = self._make_misp_bool(headerless) + query['includeSightings'] = self._make_misp_bool(include_sightings) + query['includeCorrelations'] = self._make_misp_bool(include_correlations) + url = urljoin(self.root_url, f'{controller}/restSearch') + response = self._prepare_request('POST', url, data=query) + if return_format == 'json': + normalized_response = self._check_response(response, expect_json=True) else: - url = urljoin(self.root_url, 'objects/add/{}'.format(event_id)) - response = self._prepare_request('POST', url, misp_object.to_json()) - return self._check_response(response) + normalized_response = self._check_response(response) - @deprecated(reason="Use ExpandedPyMISP.update_object", version='2.4.111', action='default') - def edit_object(self, misp_object, object_id=None): - """Edit an existing object""" - if object_id: - param = object_id - elif hasattr(misp_object, 'uuid'): - param = misp_object.uuid - elif hasattr(misp_object, 'id'): - param = misp_object.id + if return_format == 'csv' and (self.global_pythonify or pythonify) and not headerless: + return self._csv_to_dict(normalized_response) + + if 'errors' in normalized_response: + return normalized_response + + if return_format == 'json' and self.global_pythonify or pythonify: + # The response is in json, we can convert it to a list of pythonic MISP objects + to_return = [] + if controller == 'events': + for e in normalized_response: + me = MISPEvent() + me.load(e) + to_return.append(me) + elif controller == 'attributes': + # FIXME: obvs, this is hurting my soul. We need something generic. + for a in normalized_response.get('Attribute'): + ma = MISPAttribute() + ma.from_dict(**a) + if 'Event' in ma: + me = MISPEvent() + me.from_dict(**ma.Event) + ma.Event = me + if 'RelatedAttribute' in ma: + related_attributes = [] + for ra in ma.RelatedAttribute: + r_attribute = MISPAttribute() + r_attribute.from_dict(**ra) + if 'Event' in r_attribute: + me = MISPEvent() + me.from_dict(**r_attribute.Event) + r_attribute.Event = me + related_attributes.append(r_attribute) + ma.RelatedAttribute = related_attributes + if 'Sighting' in ma: + sightings = [] + for sighting in ma.Sighting: + s = MISPSighting() + s.from_dict(**sighting) + sightings.append(s) + ma.Sighting = sightings + to_return.append(ma) + elif controller == 'objects': + raise PyMISPNotImplementedYet('Not implemented yet') + return to_return + + return normalized_response + + def search_index(self, published: Optional[bool]=None, eventid: Optional[SearchType]=None, + tags: Optional[SearchParameterTypes]=None, + date_from: Optional[DateTypes]=None, + date_to: Optional[DateTypes]=None, + eventinfo: Optional[str]=None, + threatlevel: Optional[List[SearchType]]=None, + distribution: Optional[List[SearchType]]=None, + analysis: Optional[List[SearchType]]=None, + org: Optional[SearchParameterTypes]=None, + timestamp: Optional[DateInterval]=None, + pythonify: Optional[bool]=None): + """Search only at the index level. Using ! in front of a value means NOT (default is OR) + + :param published: Set whether published or unpublished events should be returned. Do not set the parameter if you want both. + :param eventid: The events that should be included / excluded from the search + :param tags: Tags to search or to exclude. You can pass a list, or the output of `build_complex_query` + :param date_from: Events with the date set to a date after the one specified. This filter will use the date of the event. + :param date_to: Events with the date set to a date before the one specified. This filter will use the date of the event. + :param eventinfo: Filter on the event's info field. + :param threatlevel: Threat level(s) (1,2,3,4) | list + :param distribution: Distribution level(s) (0,1,2,3) | list + :param analysis: Analysis level(s) (0,1,2) | list + :param org: Search by the creator organisation by supplying the organisation identifier. + :param timestamp: Restrict the results by the timestamp (last edit). Any event with a timestamp newer than the given timestamp will be returned. In case you are dealing with /attributes as scope, the attribute's timestamp will be used for the lookup. + :param pythonify: Returns a list of PyMISP Objects instead or the plain json output. Warning: it might use a lot of RAM + """ + query = locals() + query.pop('self') + query.pop('pythonify') + if query.get('date_from'): + query['datefrom'] = self._make_timestamp(query.pop('date_from')) + if query.get('date_to'): + query['dateuntil'] = self._make_timestamp(query.pop('date_to')) + + if query.get('timestamp') is not None: + timestamp = query.pop('timestamp') + if isinstance(timestamp, (list, tuple)): + query['timestamp'] = (self._make_timestamp(timestamp[0]), self._make_timestamp(timestamp[1])) + else: + query['timestamp'] = self._make_timestamp(timestamp) + + url = urljoin(self.root_url, 'events/index') + response = self._prepare_request('POST', url, data=query) + normalized_response = self._check_response(response, expect_json=True) + + if not (self.global_pythonify or pythonify): + return normalized_response + to_return = [] + for e_meta in normalized_response: + me = MISPEvent() + me.from_dict(**e_meta) + to_return.append(me) + return to_return + + def search_sightings(self, context: Optional[str]=None, + context_id: Optional[SearchType]=None, + type_sighting: Optional[str]=None, + date_from: Optional[DateTypes]=None, + date_to: Optional[DateTypes]=None, + publish_timestamp: Optional[DateInterval]=None, last: Optional[DateInterval]=None, + org: Optional[SearchType]=None, + source: Optional[str]=None, + include_attribute: Optional[bool]=None, + include_event_meta: Optional[bool]=None, + pythonify: Optional[bool]=False + ): + '''Search sightings + + :param context: The context of the search. Can be either "attribute", "event", or nothing (will then match on events and attributes). + :param context_id: Only relevant if context is either "attribute" or "event". Then it is the relevant ID. + :param type_sighting: Type of sighting + :param date_from: Events with the date set to a date after the one specified. This filter will use the date of the event. + :param date_to: Events with the date set to a date before the one specified. This filter will use the date of the event. + :param publish_timestamp: Restrict the results by the last publish timestamp (newer than). + :param org: Search by the creator organisation by supplying the organisation identifier. + :param source: Source of the sighting + :param include_attribute: Include the attribute. + :param include_event_meta: Include the meta information of the event. + + Deprecated: + + :param last: synonym for publish_timestamp + + :Example: + + >>> misp.search_sightings(publish_timestamp='30d') # search sightings for the last 30 days on the instance + [ ... ] + >>> misp.search_sightings(context='attribute', context_id=6, include_attribute=True) # return list of sighting for attribute 6 along with the attribute itself + [ ... ] + >>> misp.search_sightings(context='event', context_id=17, include_event_meta=True, org=2) # return list of sighting for event 17 filtered with org id 2 + ''' + query = {'returnFormat': 'json'} + if context is not None: + if context not in ['attribute', 'event']: + raise ValueError('context has to be in {}'.format(', '.join(['attribute', 'event']))) + url_path = f'sightings/restSearch/{context}' else: - raise PyMISPError('In order to update an object, you have to provide an object ID (either in the misp_object, or as a parameter)') - url = urljoin(self.root_url, 'objects/edit/{}'.format(param)) - response = self._prepare_request('POST', url, misp_object.to_json()) - return self._check_response(response) + url_path = 'sightings/restSearch' + if isinstance(context_id, (MISPEvent, MISPAttribute)): + context_id = self.__get_uuid_or_id_from_abstract_misp(context_id) + query['id'] = context_id + query['type'] = type_sighting + query['from'] = date_from + query['to'] = date_to + query['last'] = publish_timestamp + query['org_id'] = org + query['source'] = source + query['includeAttribute'] = include_attribute + query['includeEvent'] = include_event_meta - @deprecated(reason="Use ExpandedPyMISP.delete_object", version='2.4.111', action='default') - def delete_object(self, id): - """Deletes an object""" - url = urljoin(self.root_url, 'objects/delete/{}'.format(id)) - response = self._prepare_request('POST', url) - return self._check_response(response) + url = urljoin(self.root_url, url_path) + response = self._prepare_request('POST', url, data=query) + normalized_response = self._check_response(response, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in normalized_response: + return normalized_response - @deprecated(reason="Use ExpandedPyMISP.add_object_reference", version='2.4.111', action='default') - def add_object_reference(self, misp_object_reference): - """Add a reference to an object""" - url = urljoin(self.root_url, 'object_references/add') - response = self._prepare_request('POST', url, misp_object_reference.to_json()) - return self._check_response(response) + if self.global_pythonify or pythonify: + to_return = [] + for s in normalized_response: + entries = {} + s_data = s['Sighting'] + if include_event_meta: + e = s_data.pop('Event') + me = MISPEvent() + me.from_dict(**e) + entries['event'] = me + if include_attribute: + a = s_data.pop('Attribute') + ma = MISPAttribute() + ma.from_dict(**a) + entries['attribute'] = ma + ms = MISPSighting() + ms.from_dict(**s_data) + entries['sighting'] = ms + to_return.append(entries) + return to_return + return normalized_response - @deprecated(reason="Use ExpandedPyMISP.delete_object_reference", version='2.4.111', action='default') - def delete_object_reference(self, id): - """Deletes a reference to an object""" - url = urljoin(self.root_url, 'object_references/delete/{}'.format(id)) - response = self._prepare_request('POST', url) - return self._check_response(response) + def search_logs(self, limit: Optional[int]=None, page: Optional[int]=None, + log_id: Optional[int]=None, title: Optional[str]=None, + created: Optional[DateTypes]=None, model: Optional[str]=None, + action: Optional[str]=None, user_id: Optional[int]=None, + change: Optional[str]=None, email: Optional[str]=None, + org: Optional[str]=None, description: Optional[str]=None, + ip: Optional[str]=None, pythonify: Optional[bool]=False): + '''Search in logs - @deprecated(reason="Use ExpandedPyMISP.object_templates", version='2.4.111', action='default') - def get_object_templates_list(self): - """Returns the list of Object templates available on the MISP instance""" - url = urljoin(self.root_url, 'objectTemplates') - response = self._prepare_request('GET', url) - return self._check_response(response) + Note: to run substring queries simply append/prepend/encapsulate the search term with % + + :param limit: Limit the number of results returned, depending on the scope (for example 10 attributes or 10 full events). + :param page: If a limit is set, sets the page to be returned. page 3, limit 100 will return records 201->300). + :param log_id: Log ID + :param title: Log Title + :param created: Creation timestamp + :param model: Model name that generated the log entry + :param action: The thing that was done + :param user_id: ID of the user doing the action + :param change: Change that occured + :param email: Email of the user + :param org: Organisation of the User doing the action + :param description: Description of the action + :param ip: Origination IP of the User doing the action + :param pythonify: Returns a list of PyMISP Objects instead or the plain json output. Warning: it might use a lot of RAM + ''' + query = locals() + query.pop('self') + query.pop('pythonify') + if log_id is not None: + query['id'] = query.pop('log_id') + + response = self._prepare_request('POST', 'admin/logs/index', data=query) + normalized_response = self._check_response(response, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in normalized_response: + return normalized_response + + to_return = [] + for l in normalized_response: + ml = MISPLog() + ml.from_dict(**l) + to_return.append(ml) + return to_return + + def search_feeds(self, value: Optional[SearchParameterTypes]=None, pythonify: Optional[bool]=False): + '''Search in the feeds cached on the servers''' + response = self._prepare_request('POST', '/feeds/searchCaches', data={'value': value}) + normalized_response = self._check_response(response, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in normalized_response: + return normalized_response + to_return = [] + for feed in normalized_response: + f = MISPFeed() + f.from_dict(**feed) + to_return.append(f) + return to_return + + # ## END Search methods ### + + # ## BEGIN Communities ### + + def communities(self, pythonify: bool=False): + """Get all the communities.""" + communities = self._prepare_request('GET', 'communities') + communities = self._check_response(communities, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in communities: + return communities + to_return = [] + for community in communities: + c = MISPCommunity() + c.from_dict(**community) + to_return.append(c) + return to_return + + def get_community(self, community: Union[MISPCommunity, int, str, UUID], pythonify: bool=False): + '''Get an community from a MISP instance''' + community_id = self.__get_uuid_or_id_from_abstract_misp(community) + community = self._prepare_request('GET', f'communities/view/{community_id}') + community = self._check_response(community, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in community: + return community + c = MISPCommunity() + c.from_dict(**community) + return c + + def request_community_access(self, community: Union[MISPCommunity, int, str, UUID], + requestor_email_address: str=None, + requestor_gpg_key: str=None, + requestor_organisation_name: str=None, + requestor_organisation_uuid: str=None, + requestor_organisation_description: str=None, + message: str=None, sync: bool=False, + anonymise_requestor_server: bool=False, + mock: bool=False): + community_id = self.__get_uuid_or_id_from_abstract_misp(community) + to_post = {'org_name': requestor_organisation_name, + 'org_uuid': requestor_organisation_uuid, + 'org_description': requestor_organisation_description, + 'email': requestor_email_address, 'gpgkey': requestor_gpg_key, + 'message': message, 'anonymise': anonymise_requestor_server, 'sync': sync, + 'mock': mock} + r = self._prepare_request('POST', f'communities/requestAccess/{community_id}', data=to_post) + return self._check_response(r, expect_json=True) + + # ## END Communities ### + + # ## BEGIN Event Delegation ### + + def event_delegations(self, pythonify: bool=False): + """Get all the event delegations.""" + delegations = self._prepare_request('GET', 'event_delegations') + delegations = self._check_response(delegations, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in delegations: + return delegations + to_return = [] + for delegation in delegations: + d = MISPEventDelegation() + d.from_dict(**delegation) + to_return.append(d) + return to_return + + def accept_event_delegation(self, delegation: Union[MISPEventDelegation, int, str], pythonify: bool=False): + delegation_id = self.__get_uuid_or_id_from_abstract_misp(delegation) + delegation = self._prepare_request('POST', f'event_delegations/acceptDelegation/{delegation_id}') + return self._check_response(delegation, expect_json=True) + + def discard_event_delegation(self, delegation: Union[MISPEventDelegation, int, str], pythonify: bool=False): + delegation_id = self.__get_uuid_or_id_from_abstract_misp(delegation) + delegation = self._prepare_request('POST', f'event_delegations/deleteDelegation/{delegation_id}') + return self._check_response(delegation, expect_json=True) + + def delegate_event(self, event: Union[MISPEvent, int, str, UUID]=None, + organisation: Union[MISPOrganisation, int, str, UUID]=None, + event_delegation: MISPEventDelegation=None, + distribution: int=-1, message: str='', pythonify: bool=False): + '''Note: distribution == -1 means recipient decides''' + if event and organisation: + event_id = self.__get_uuid_or_id_from_abstract_misp(event) + organisation_id = self.__get_uuid_or_id_from_abstract_misp(organisation) + data = {'event_id': event_id, 'org_id': organisation_id, 'distribution': distribution, 'message': message} + elif event_delegation: + data = event_delegation + else: + raise PyMISPError('Either event and organisation OR event_delegation are required.') + delegation = self._prepare_request('POST', f'event_delegations/delegateEvent/{event_id}', data=data) + delegation = self._check_response(delegation, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in delegation: + return delegation + d = MISPEventDelegation() + d.from_dict(**delegation) + return d + + # ## END Event Delegation ### + + # ## BEGIN Others ### + + def push_event_to_ZMQ(self, event: Union[MISPEvent, int, str, UUID]): + """Force push an event on ZMQ""" + event_id = self.__get_uuid_or_id_from_abstract_misp(event) + response = self._prepare_request('POST', f'events/pushEventToZMQ/{event_id}.json') + return self._check_response(response, expect_json=True) + + def direct_call(self, url: str, data: dict=None, params: dict={}, kw_params: dict={}): + '''Very lightweight call that posts a data blob (python dictionary or json string) on the URL''' + if data is None: + response = self._prepare_request('GET', url, params=params, kw_params=kw_params) + else: + response = self._prepare_request('POST', url, data=data, params=params, kw_params=kw_params) + return self._check_response(response, lenient_response_type=True) + + def freetext(self, event: Union[MISPEvent, int, str, UUID], string: str, adhereToWarninglists: Union[bool, str]=False, + distribution: int=None, returnMetaAttributes: bool=False, pythonify: bool=False, **kwargs): + """Pass a text to the freetext importer""" + event_id = self.__get_uuid_or_id_from_abstract_misp(event) + query = {"value": string} + wl_params = [False, True, 'soft'] + if adhereToWarninglists in wl_params: + query['adhereToWarninglists'] = adhereToWarninglists + else: + raise PyMISPError('Invalid parameter, adhereToWarninglists Can only be {}'.format(', '.join(wl_params))) + if distribution is not None: + query['distribution'] = distribution + if returnMetaAttributes: + query['returnMetaAttributes'] = returnMetaAttributes + attributes = self._prepare_request('POST', f'events/freeTextImport/{event_id}', data=query, **kwargs) + attributes = self._check_response(attributes, expect_json=True) + if returnMetaAttributes or not (self.global_pythonify or pythonify) or 'errors' in attributes: + return attributes + to_return = [] + for attribute in attributes: + a = MISPAttribute() + a.from_dict(**attribute) + to_return.append(a) + return to_return + + def upload_stix(self, path, version: str='2'): + """Upload a STIX file to MISP. + :param path: Path to the STIX on the disk (can be a path-like object, or a pseudofile) + :param version: Can be 1 or 2 + """ + if isinstance(path, (str, Path)): + with open(path, 'rb') as f: + to_post = f.read() + else: + to_post = path.read() + + if isinstance(to_post, bytes): + to_post = to_post.decode() + + if str(version) == '1': + url = urljoin(self.root_url, '/events/upload_stix') + response = self._prepare_request('POST', url, data=to_post, output_type='xml') + else: + url = urljoin(self.root_url, '/events/upload_stix/2') + response = self._prepare_request('POST', url, data=to_post) - @deprecated(reason="Use ExpandedPyMISP.get_object_template", version='2.4.111', action='default') - def get_object_template(self, object_uuid): - """Gets the full object template corresponting the UUID passed as parameter""" - url = urljoin(self.root_url, 'objectTemplates/view/{}'.format(object_uuid)) - response = self._prepare_request('GET', url) - return self._check_response(response) - if 'ObjectTemplate' in response: - return response['ObjectTemplate']['id'] return response - @deprecated(reason="Use ExpandedPyMISP.get_object_template - open an issue if you really need this method.", version='2.4.111', action='default') - def get_object_template_id(self, object_uuid): - """Gets the template ID corresponting the UUID passed as parameter""" - template = self.get_object_template(object_uuid) - if 'ObjectTemplate' in template: - return template['ObjectTemplate']['id'] - # Contains the error message. - return template + # ## END Others ### - @deprecated(reason="Use ExpandedPyMISP.update_object_templates", version='2.4.111', action='default') - def update_object_templates(self): - url = urljoin(self.root_url, 'objectTemplates/update') - response = self._prepare_request('POST', url) - return self._check_response(response) + # ## BEGIN Statistics ### - # ########################### - # ####### Deprecated ######## - # ########################### - - @deprecated(reason="Use ExpandedPyMISP.tag", version='2.4.111', action='default') - def add_tag(self, event, tag, attribute=False): - if attribute: - to_post = {'request': {'Attribute': {'id': event['id'], 'tag': tag}}} - path = 'attributes/addTag' + def attributes_statistics(self, context: str='type', percentage: bool=False): + """Get attributes statistics from the MISP instance.""" + # FIXME: https://github.com/MISP/MISP/issues/4874 + if context not in ['type', 'category']: + raise PyMISPError('context can only be "type" or "category"') + if percentage: + path = f'attributes/attributeStatistics/{context}/true' else: - # Allow for backwards-compat with old style - if "Event" in event: - event = event["Event"] - to_post = {'request': {'Event': {'id': event['id'], 'tag': tag}}} - path = 'events/addTag' - url = urljoin(self.root_url, path) - response = self._prepare_request('POST', url, json.dumps(to_post)) + path = f'attributes/attributeStatistics/{context}' + response = self._prepare_request('GET', path) + return self._check_response(response, expect_json=True) + + def tags_statistics(self, percentage: bool=False, name_sort: bool=False): + """Get tags statistics from the MISP instance""" + # FIXME: https://github.com/MISP/MISP/issues/4874 + # NOTE: https://github.com/MISP/MISP/issues/4879 + if percentage: + percentage = 'true' + else: + percentage = 'false' + if name_sort: + name_sort = 'true' + else: + name_sort = 'false' + response = self._prepare_request('GET', f'tags/tagStatistics/{percentage}/{name_sort}') return self._check_response(response) - @deprecated(reason="Use ExpandedPyMISP.untag", version='2.4.111', action='default') - def remove_tag(self, event, tag, attribute=False): - if attribute: - to_post = {'request': {'Attribute': {'id': event['id'], 'tag': tag}}} - path = 'attributes/removeTag' - else: - to_post = {'request': {'Event': {'id': event['Event']['id'], 'tag': tag}}} - path = 'events/removeTag' - url = urljoin(self.root_url, path) - response = self._prepare_request('POST', url, json.dumps(to_post)) + def users_statistics(self, context: str='data'): + """Get users statistics from the MISP instance""" + availables_contexts = ['data', 'orgs', 'users', 'tags', 'attributehistogram', 'sightings', 'galaxyMatrix'] + if context not in availables_contexts: + raise PyMISPError("context can only be {','.join(availables_contexts)}") + response = self._prepare_request('GET', f'users/statistics/{context}') return self._check_response(response) + + # ## END Statistics ### + + # ## BEGIN User Settings ### + + def user_settings(self, pythonify: bool=False): + """Get all the user settings.""" + user_settings = self._prepare_request('GET', 'user_settings') + user_settings = self._check_response(user_settings, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in user_settings: + return user_settings + to_return = [] + for user_setting in user_settings: + u = MISPUserSetting() + u.from_dict(**user_setting) + to_return.append(u) + return to_return + + def get_user_setting(self, user_setting: str, user: Union[MISPUser, int, str, UUID]=None, pythonify: bool=False): + '''Get an user setting''' + query = {'setting': user_setting} + if user: + query['user_id'] = self.__get_uuid_or_id_from_abstract_misp(user) + response = self._prepare_request('POST', f'user_settings/getSetting') + user_setting = self._check_response(response, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in user_setting: + return user_setting + u = MISPUserSetting() + u.from_dict(**user_setting) + return u + + def set_user_setting(self, user_setting: str, value: Union[str, dict], user: Union[MISPUser, int, str, UUID]=None, pythonify: bool=False): + '''Get an user setting''' + query = {'setting': user_setting} + if isinstance(value, dict): + value = json.dumps(value) + query['value'] = value + if user: + query['user_id'] = self.__get_uuid_or_id_from_abstract_misp(user) + response = self._prepare_request('POST', f'user_settings/setSetting', data=query) + user_setting = self._check_response(response, expect_json=True) + if not (self.global_pythonify or pythonify) or 'errors' in user_setting: + return user_setting + u = MISPUserSetting() + u.from_dict(**user_setting) + return u + + def delete_user_setting(self, user_setting: str, user: Union[MISPUser, int, str, UUID]=None): + '''Delete a user setting''' + query = {'setting': user_setting} + if user: + query['user_id'] = self.__get_uuid_or_id_from_abstract_misp(user) + response = self._prepare_request('POST', f'user_settings/delete', data=query) + return self._check_response(response, expect_json=True) + + # ## END User Settings ### + + # ## BEGIN Global helpers ### + + def change_sharing_group_on_entity(self, misp_entity: AbstractMISP, sharing_group_id, pythonify: bool=False): + """Change the sharing group of an event, an attribute, or an object""" + misp_entity.distribution = 4 # Needs to be 'Sharing group' + if 'SharingGroup' in misp_entity: # Delete former SharingGroup information + del misp_entity.SharingGroup + misp_entity.sharing_group_id = sharing_group_id # Set new sharing group id + if isinstance(misp_entity, MISPEvent): + return self.update_event(misp_entity, pythonify=pythonify) + + if isinstance(misp_entity, MISPObject): + return self.update_object(misp_entity, pythonify=pythonify) + + if isinstance(misp_entity, MISPAttribute): + return self.update_attribute(misp_entity, pythonify=pythonify) + + raise PyMISPError('The misp_entity must be MISPEvent, MISPObject or MISPAttribute') + + def tag(self, misp_entity: Union[AbstractMISP, str], tag: Union[MISPTag, str], local: bool=False): + """Tag an event or an attribute. misp_entity can be a MISPEvent, a MISP Attribute, or a UUID""" + if 'uuid' in misp_entity: + uuid = misp_entity.uuid + else: + uuid = misp_entity + if isinstance(tag, MISPTag): + tag = tag.name + to_post = {'uuid': uuid, 'tag': tag, 'local': local} + response = self._prepare_request('POST', 'tags/attachTagToObject', data=to_post) + return self._check_response(response, expect_json=True) + + def untag(self, misp_entity: Union[AbstractMISP, str], tag: Union[MISPTag, str]): + """Untag an event or an attribute. misp_entity can be a UUID""" + if 'uuid' in misp_entity: + uuid = misp_entity.uuid + else: + uuid = misp_entity + if isinstance(tag, MISPTag): + tag = tag.name + to_post = {'uuid': uuid, 'tag': tag} + response = self._prepare_request('POST', 'tags/removeTagFromObject', data=to_post) + return self._check_response(response, expect_json=True) + + def build_complex_query(self, or_parameters: Optional[List[SearchType]]=None, + and_parameters: Optional[List[SearchType]]=None, + not_parameters: Optional[List[SearchType]]=None): + '''Build a complex search query. MISP expects a dictionary with AND, OR and NOT keys.''' + to_return = {} + if and_parameters: + to_return['AND'] = and_parameters + if not_parameters: + to_return['NOT'] = not_parameters + if or_parameters: + to_return['OR'] = or_parameters + return to_return + + # ## END Global helpers ### + + # ## Internal methods ### + + def _old_misp(self, minimal_version_required: tuple, removal_date: Union[str, date, datetime], method: str=None, message: str=None): + if self._misp_version >= minimal_version_required: + return False + if isinstance(removal_date, (datetime, date)): + removal_date = removal_date.isoformat() + to_print = f'The instance of MISP you are using is outdated. Unless you update your MISP instance, {method} will stop working after {removal_date}.' + if message: + to_print += f' {message}' + warnings.warn(to_print, DeprecationWarning) + return True + + def __get_uuid_or_id_from_abstract_misp(self, obj: Union[AbstractMISP, int, str, UUID]): + if isinstance(obj, UUID): + return str(obj) + if isinstance(obj, (int, str)): + return obj + + if isinstance(obj, dict) and len(obj.keys()) == 1: + # We have an object in that format: {'Event': {'id': 2, ...}} + # We need to get the content of that dictionary + obj = obj[list(obj.keys())[0]] + + if isinstance(obj, MISPShadowAttribute): + # A ShadowAttribute has the same UUID as the related Attribute, we *need* to use the ID + return obj['id'] + if isinstance(obj, MISPEventDelegation): + # An EventDelegation doesn't have a uuid, we *need* to use the ID + return obj['id'] + if 'uuid' in obj: + return obj['uuid'] + return obj['id'] + + def _make_misp_bool(self, parameter: Union[bool, str, None]): + '''MISP wants 0 or 1 for bool, so we avoid True/False '0', '1' ''' + if parameter is None: + return 0 + return 1 if int(parameter) else 0 + + def _make_timestamp(self, value: DateTypes): + '''Catch-all method to normalize anything that can be converted to a timestamp''' + if isinstance(value, datetime): + return value.timestamp() + + if isinstance(value, date): + return datetime.combine(value, datetime.max.time()).timestamp() + + if isinstance(value, str): + if value.isdigit(): + return value + try: + float(value) + return value + except ValueError: + # The value can also be '1d', '10h', ... + return value + return value + + def _check_response(self, response, lenient_response_type=False, expect_json=False): + """Check if the response from the server is not an unexpected error""" + if response.status_code >= 500: + logger.critical(everything_broken.format(response.request.headers, response.request.body, response.text)) + raise MISPServerError(f'Error code 500:\n{response.text}') + + if 400 <= response.status_code < 500: + # The server returns a json message with the error details + try: + error_message = response.json() + except Exception: + raise MISPServerError(f'Error code {response.status_code}:\n{response.text}') + + logger.error(f'Something went wrong ({response.status_code}): {error_message}') + return {'errors': (response.status_code, error_message)} + + # At this point, we had no error. + + try: + response = response.json() + if logger.isEnabledFor(logging.DEBUG): + logger.debug(response) + if isinstance(response, dict) and response.get('response') is not None: + # Cleanup. + response = response['response'] + return response + except Exception: + if logger.isEnabledFor(logging.DEBUG): + logger.debug(response.text) + if expect_json: + raise PyMISPUnexpectedResponse(f'Unexpected response from server: {response.text}') + if lenient_response_type and not response.headers.get('content-type').startswith('application/json'): + return response.text + if not response.content: + # Empty response + logger.error('Got an empty response.') + return {'errors': 'The response is empty.'} + return response.text + + def __repr__(self): + return f'<{self.__class__.__name__}(url={self.root_url})' + + def _prepare_request(self, request_type: str, url: str, data: dict={}, params: dict={}, + kw_params: dict={}, output_type: str='json'): + '''Prepare a request for python-requests''' + url = urljoin(self.root_url, url) + if data: + if not isinstance(data, str): # Else, we already have a text blob to send + if isinstance(data, dict): # Else, we can directly json encode. + # Remove None values. + data = {k: v for k, v in data.items() if v is not None} + data = json.dumps(data, default=pymisp_json_default) + + if logger.isEnabledFor(logging.DEBUG): + logger.debug(f'{request_type} - {url}') + if data is not None: + logger.debug(data) + + if kw_params: + # CakePHP params in URL + to_append_url = '/'.join([f'{k}:{v}' for k, v in kw_params.items()]) + url = f'{url}/{to_append_url}' + req = requests.Request(request_type, url, data=data, params=params) + with requests.Session() as s: + user_agent = f'PyMISP {__version__} - Python {".".join(str(x) for x in sys.version_info[:2])}' + if self.tool: + user_agent = f'{user_agent} - {self.tool}' + req.auth = self.auth + prepped = s.prepare_request(req) + prepped.headers.update( + {'Authorization': self.key, + 'Accept': f'application/{output_type}', + 'content-type': f'application/{output_type}', + 'User-Agent': user_agent}) + if logger.isEnabledFor(logging.DEBUG): + logger.debug(prepped.headers) + settings = s.merge_environment_settings(req.url, proxies=self.proxies or {}, stream=None, verify=self.ssl, cert=self.cert) + return s.send(prepped, **settings) + + def _csv_to_dict(self, csv_content: str): + '''Makes a list of dict out of a csv file (requires headers)''' + fieldnames, lines = csv_content.split('\n', 1) + fieldnames = fieldnames.split(',') + to_return = [] + for line in csv.reader(lines.split('\n')): + if line: + to_return.append({fname: value for fname, value in zip(fieldnames, line)}) + return to_return diff --git a/pymisp/aping.py b/pymisp/aping.py deleted file mode 100644 index d26d1ce..0000000 --- a/pymisp/aping.py +++ /dev/null @@ -1,2243 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -from typing import TypeVar, Optional, Tuple, List, Dict, Union -from datetime import date, datetime -import csv -from pathlib import Path -import logging -from urllib.parse import urljoin -import json -import requests -from requests.auth import AuthBase -import re -from uuid import UUID -import warnings -import sys - -from . import __version__ -from .exceptions import MISPServerError, PyMISPUnexpectedResponse, PyMISPNotImplementedYet, PyMISPError, NoURL, NoKey -from .api import everything_broken, PyMISP -from .mispevent import MISPEvent, MISPAttribute, MISPSighting, MISPLog, MISPObject, \ - MISPUser, MISPOrganisation, MISPShadowAttribute, MISPWarninglist, MISPTaxonomy, \ - MISPGalaxy, MISPNoticelist, MISPObjectReference, MISPObjectTemplate, MISPSharingGroup, \ - MISPRole, MISPServer, MISPFeed, MISPEventDelegation, MISPCommunity, MISPUserSetting -from .abstract import pymisp_json_default, MISPTag, AbstractMISP, describe_types - -SearchType = TypeVar('SearchType', str, int) -# str: string to search / list: values to search (OR) / dict: {'OR': [list], 'NOT': [list], 'AND': [list]} -SearchParameterTypes = TypeVar('SearchParameterTypes', str, List[SearchType], Dict[str, SearchType]) -DateTypes = TypeVar('DateTypes', datetime, date, SearchType, float) -DateInterval = TypeVar('DateInterval', DateTypes, Tuple[DateTypes, DateTypes]) - -ToIDSType = TypeVar('ToIDSType', str, int, bool) - -logger = logging.getLogger('pymisp') - - -class ExpandedPyMISP(PyMISP): - """Python API for MISP - - :param url: URL of the MISP instance you want to connect to - :param key: API key of the user you want to use - :param ssl: can be True or False (to check ot not the validity of the certificate. Or a CA_BUNDLE in case of self signed certificate (the concatenation of all the *.crt of the chain) - :param debug: Write all the debug information to stderr - :param proxies: Proxy dict as describes here: http://docs.python-requests.org/en/master/user/advanced/#proxies - :param cert: Client certificate, as described there: http://docs.python-requests.org/en/master/user/advanced/#client-side-certificates - :param auth: The auth parameter is passed directly to requests, as described here: http://docs.python-requests.org/en/master/user/authentication/ - :param tool: The software using PyMISP (string), used to set a unique user-agent - """ - - def __init__(self, url: str, key: str, ssl=True, debug: bool=False, proxies: dict={}, - cert: Tuple[str, tuple]=None, auth: AuthBase=None, tool: str=''): - if not url: - raise NoURL('Please provide the URL of your MISP instance.') - if not key: - raise NoKey('Please provide your authorization key.') - - self.root_url = url - self.key = key - self.ssl = ssl - self.proxies = proxies - self.cert = cert - self.auth = auth - self.tool = tool - - self.global_pythonify = False - - self.resources_path = Path(__file__).parent / 'data' - if debug: - logger.setLevel(logging.DEBUG) - logger.info('To configure logging in your script, leave it to None and use the following: import logging; logging.getLogger(\'pymisp\').setLevel(logging.DEBUG)') - - try: - # Make sure the MISP instance is working and the URL is valid - response = self.recommended_pymisp_version - if response.get('errors'): - logger.warning(response.get('errors')[0]) - else: - pymisp_version_tup = tuple(int(x) for x in __version__.split('.')) - recommended_version_tup = tuple(int(x) for x in response['version'].split('.')) - if recommended_version_tup < pymisp_version_tup[:3]: - logger.info(f"The version of PyMISP recommended by the MISP instance (response['version']) is older than the one you're using now ({__version__}). If you have a problem, please upgrade the MISP instance or use an older PyMISP version.") - elif pymisp_version_tup[:3] < recommended_version_tup: - logger.warning(f"The version of PyMISP recommended by the MISP instance ({response['version']}) is newer than the one you're using now ({__version__}). Please upgrade PyMISP.") - - misp_version = self.misp_instance_version - if 'version' in misp_version: - self._misp_version = tuple(int(v) for v in misp_version['version'].split('.')) - - # Get the user information - self._current_user, self._current_role, self._current_user_settings = self.get_user(pythonify=True, expanded=True) - except Exception as e: - raise PyMISPError(f'Unable to connect to MISP ({self.root_url}). Please make sure the API key and the URL are correct (http/https is required): {e}') - - try: - self.describe_types = self.describe_types_remote - except Exception: - self.describe_types = self.describe_types_local - - self.categories = self.describe_types['categories'] - self.types = self.describe_types['types'] - self.category_type_mapping = self.describe_types['category_type_mappings'] - self.sane_default = self.describe_types['sane_defaults'] - - def remote_acl(self, debug_type: str='findMissingFunctionNames'): - """This should return an empty list, unless the ACL is outdated. - debug_type can only be printAllFunctionNames, findMissingFunctionNames, or printRoleAccess - """ - response = self._prepare_request('GET', f'events/queryACL/{debug_type}') - return self._check_response(response, expect_json=True) - - @property - def describe_types_local(self): - '''Returns the content of describe types from the package''' - return describe_types - - @property - def describe_types_remote(self): - '''Returns the content of describe types from the remote instance''' - response = self._prepare_request('GET', 'attributes/describeTypes.json') - remote_describe_types = self._check_response(response, expect_json=True) - return remote_describe_types['result'] - - @property - def recommended_pymisp_version(self): - """Returns the recommended API version from the server""" - response = self._prepare_request('GET', 'servers/getPyMISPVersion.json') - return self._check_response(response, expect_json=True) - - @property - def version(self): - """Returns the version of PyMISP you're curently using""" - return {'version': __version__} - - @property - def pymisp_version_master(self): - """Get the most recent version of PyMISP from github""" - r = requests.get('https://raw.githubusercontent.com/MISP/PyMISP/master/pymisp/__init__.py') - if r.status_code == 200: - version = re.findall("__version__ = '(.*)'", r.text) - return {'version': version[0]} - return {'error': 'Impossible to retrieve the version of the master branch.'} - - @property - def misp_instance_version(self): - """Returns the version of the instance.""" - response = self._prepare_request('GET', 'servers/getVersion.json') - return self._check_response(response, expect_json=True) - - @property - def misp_instance_version_master(self): - """Get the most recent version from github""" - r = requests.get('https://raw.githubusercontent.com/MISP/MISP/2.4/VERSION.json') - if r.status_code == 200: - master_version = json.loads(r.text) - return {'version': '{}.{}.{}'.format(master_version['major'], master_version['minor'], master_version['hotfix'])} - return {'error': 'Impossible to retrieve the version of the master branch.'} - - def update_misp(self): - response = self._prepare_request('POST', '/servers/update') - if self._old_misp((2, 4, 116), '2020-01-01', sys._getframe().f_code.co_name): - return self._check_response(response, lenient_response_type=True) - return self._check_response(response, expect_json=True) - - def set_server_setting(self, setting: str, value: Union[str, int, bool], force: bool=False): - data = {'value': value, 'force': force} - response = self._prepare_request('POST', f'/servers/serverSettingsEdit/{setting}', data=data) - return self._check_response(response, expect_json=True) - - def get_server_setting(self, setting: str): - response = self._prepare_request('GET', f'/servers/getSetting/{setting}') - return self._check_response(response, expect_json=True) - - def server_settings(self): - response = self._prepare_request('GET', f'/servers/serverSettings') - return self._check_response(response, expect_json=True) - - def restart_workers(self): - response = self._prepare_request('POST', f'/servers/restartWorkers') - return self._check_response(response, expect_json=True) - - def db_schema_diagnostic(self): - response = self._prepare_request('GET', f'/servers/dbSchemaDiagnostic') - return self._check_response(response, expect_json=True) - - def toggle_global_pythonify(self): - self.global_pythonify = not self.global_pythonify - - # ## BEGIN Event ## - - def events(self, pythonify: bool=False): - events = self._prepare_request('GET', 'events') - events = self._check_response(events, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in events: - return events - to_return = [] - for event in events: - e = MISPEvent() - e.from_dict(**event) - to_return.append(e) - return to_return - - def get_event(self, event: Union[MISPEvent, int, str, UUID], deleted: [bool, int, list]=False, pythonify: bool=False): - '''Get an event from a MISP instance''' - event_id = self.__get_uuid_or_id_from_abstract_misp(event) - if deleted: - data = {'deleted': deleted} - event = self._prepare_request('POST', f'events/view/{event_id}', data=data) - else: - event = self._prepare_request('GET', f'events/view/{event_id}') - event = self._check_response(event, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in event: - return event - e = MISPEvent() - e.load(event) - return e - - def add_event(self, event: MISPEvent, pythonify: bool=False): - '''Add a new event on a MISP instance''' - new_event = self._prepare_request('POST', 'events', data=event) - new_event = self._check_response(new_event, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in new_event: - return new_event - e = MISPEvent() - e.load(new_event) - return e - - def update_event(self, event: MISPEvent, event_id: int=None, pythonify: bool=False): - '''Update an event on a MISP instance''' - if event_id is None: - event_id = self.__get_uuid_or_id_from_abstract_misp(event) - else: - event_id = self.__get_uuid_or_id_from_abstract_misp(event_id) - updated_event = self._prepare_request('POST', f'events/{event_id}', data=event) - updated_event = self._check_response(updated_event, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in updated_event: - return updated_event - e = MISPEvent() - e.load(updated_event) - return e - - def delete_event(self, event: Union[MISPEvent, int, str, UUID]): - '''Delete an event from a MISP instance''' - event_id = self.__get_uuid_or_id_from_abstract_misp(event) - response = self._prepare_request('DELETE', f'events/delete/{event_id}') - return self._check_response(response, expect_json=True) - - def publish(self, event: Union[MISPEvent, int, str, UUID], alert: bool=False): - """Publish the event with one single HTTP POST. - The default is to not send a mail as it is assumed this method is called on update. - """ - event_id = self.__get_uuid_or_id_from_abstract_misp(event) - if alert: - response = self._prepare_request('POST', f'events/alert/{event_id}') - else: - response = self._prepare_request('POST', f'events/publish/{event_id}') - return self._check_response(response, expect_json=True) - - def contact_event_reporter(self, event: Union[MISPEvent, int, str, UUID], message: str): - """Send a message to the reporter of an event""" - event_id = self.__get_uuid_or_id_from_abstract_misp(event) - to_post = {'message': message} - response = self._prepare_request('POST', f'events/contact/{event_id}', data=to_post) - return self._check_response(response, expect_json=True) - - # ## END Event ### - - # ## BEGIN Object ### - - def get_object(self, misp_object: Union[MISPObject, int, str, UUID], pythonify: bool=False): - '''Get an object from the remote MISP instance''' - object_id = self.__get_uuid_or_id_from_abstract_misp(misp_object) - misp_object = self._prepare_request('GET', f'objects/view/{object_id}') - misp_object = self._check_response(misp_object, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in misp_object: - return misp_object - o = MISPObject(misp_object['Object']['name']) - o.from_dict(**misp_object) - return o - - def add_object(self, event: Union[MISPEvent, int, str, UUID], misp_object: MISPObject, pythonify: bool=False): - '''Add a MISP Object to an existing MISP event''' - event_id = self.__get_uuid_or_id_from_abstract_misp(event) - new_object = self._prepare_request('POST', f'objects/add/{event_id}', data=misp_object) - new_object = self._check_response(new_object, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in new_object: - return new_object - o = MISPObject(new_object['Object']['name']) - o.from_dict(**new_object) - return o - - def update_object(self, misp_object: MISPObject, object_id: int=None, pythonify: bool=False): - '''Update an object on a MISP instance''' - if object_id is None: - object_id = self.__get_uuid_or_id_from_abstract_misp(misp_object) - else: - object_id = self.__get_uuid_or_id_from_abstract_misp(object_id) - updated_object = self._prepare_request('POST', f'objects/edit/{object_id}', data=misp_object) - updated_object = self._check_response(updated_object, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in updated_object: - return updated_object - o = MISPObject(updated_object['Object']['name']) - o.from_dict(**updated_object) - return o - - def delete_object(self, misp_object: Union[MISPObject, int, str, UUID]): - '''Delete an object from a MISP instance''' - object_id = self.__get_uuid_or_id_from_abstract_misp(misp_object) - response = self._prepare_request('POST', f'objects/delete/{object_id}') - return self._check_response(response, expect_json=True) - - def add_object_reference(self, misp_object_reference: MISPObjectReference, pythonify: bool=False): - """Add a reference to an object""" - object_reference = self._prepare_request('POST', 'object_references/add', misp_object_reference) - object_reference = self._check_response(object_reference, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in object_reference: - return object_reference - r = MISPObjectReference() - r.from_dict(**object_reference) - return r - - def delete_object_reference(self, object_reference: Union[MISPObjectReference, int, str, UUID]): - """Delete a reference to an object""" - object_reference_id = self.__get_uuid_or_id_from_abstract_misp(object_reference) - response = self._prepare_request('POST', f'object_references/delete/{object_reference_id}') - return self._check_response(response, expect_json=True) - - # Object templates - - def object_templates(self, pythonify: bool=False): - """Get all the object templates.""" - object_templates = self._prepare_request('GET', 'objectTemplates') - object_templates = self._check_response(object_templates, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in object_templates: - return object_templates - to_return = [] - for object_template in object_templates: - o = MISPObjectTemplate() - o.from_dict(**object_template) - to_return.append(o) - return to_return - - def get_object_template(self, object_template: Union[MISPObjectTemplate, int, str, UUID], pythonify: bool=False): - """Gets the full object template corresponting the UUID passed as parameter""" - object_template_id = self.__get_uuid_or_id_from_abstract_misp(object_template) - object_template = self._prepare_request('GET', f'objectTemplates/view/{object_template_id}') - object_template = self._check_response(object_template, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in object_template: - return object_template - t = MISPObjectTemplate() - t.from_dict(**object_template) - return t - - def update_object_templates(self): - """Trigger an update of the object templates""" - response = self._prepare_request('POST', 'objectTemplates/update') - return self._check_response(response, expect_json=True) - - # ## END Object ### - - # ## BEGIN Attribute ### - - def attributes(self, pythonify: bool=False): - attributes = self._prepare_request('GET', f'attributes/index') - attributes = self._check_response(attributes, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in attributes: - return attributes - to_return = [] - for attribute in attributes: - a = MISPAttribute() - a.from_dict(**attribute) - to_return.append(a) - return to_return - - def get_attribute(self, attribute: Union[MISPAttribute, int, str, UUID], pythonify: bool=False): - '''Get an attribute from a MISP instance''' - attribute_id = self.__get_uuid_or_id_from_abstract_misp(attribute) - attribute = self._prepare_request('GET', f'attributes/view/{attribute_id}') - attribute = self._check_response(attribute, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in attribute: - return attribute - a = MISPAttribute() - a.from_dict(**attribute) - return a - - def add_attribute(self, event: Union[MISPEvent, int, str, UUID], attribute: MISPAttribute, pythonify: bool=False): - '''Add an attribute to an existing MISP event - NOTE MISP 2.4.113+: you can pass a list of attributes. - In that case, the pythonified response is the following: {'attributes': [MISPAttribute], 'errors': {errors by attributes}}''' - event_id = self.__get_uuid_or_id_from_abstract_misp(event) - new_attribute = self._prepare_request('POST', f'attributes/add/{event_id}', data=attribute) - new_attribute = self._check_response(new_attribute, expect_json=True) - if isinstance(attribute, list): - # Multiple attributes were passed at once, the handling is totally different - if self._old_misp((2, 4, 113), '2020-01-01', sys._getframe().f_code.co_name): - return new_attribute - if not (self.global_pythonify or pythonify): - return new_attribute - to_return = {'attributes': []} - if 'errors' in new_attribute: - to_return['errors'] = new_attribute['errors'] - - for new_attr in new_attribute['Attribute']: - a = MISPAttribute() - a.from_dict(**new_attr) - to_return['attributes'].append(a) - return to_return - - if ('errors' in new_attribute and new_attribute['errors'][0] == 403 - and new_attribute['errors'][1]['message'] == 'You do not have permission to do that.'): - # At this point, we assume the user tried to add an attribute on an event they don't own - # Re-try with a proposal - return self.add_attribute_proposal(event_id, attribute, pythonify) - if not (self.global_pythonify or pythonify) or 'errors' in new_attribute: - return new_attribute - a = MISPAttribute() - a.from_dict(**new_attribute) - return a - - def update_attribute(self, attribute: MISPAttribute, attribute_id: int=None, pythonify: bool=False): - '''Update an attribute on a MISP instance''' - if attribute_id is None: - attribute_id = self.__get_uuid_or_id_from_abstract_misp(attribute) - else: - attribute_id = self.__get_uuid_or_id_from_abstract_misp(attribute_id) - updated_attribute = self._prepare_request('POST', f'attributes/edit/{attribute_id}', data=attribute) - updated_attribute = self._check_response(updated_attribute, expect_json=True) - if 'errors' in updated_attribute: - if (updated_attribute['errors'][0] == 403 - and updated_attribute['errors'][1]['message'] == 'You do not have permission to do that.'): - # At this point, we assume the user tried to update an attribute on an event they don't own - # Re-try with a proposal - return self.update_attribute_proposal(attribute_id, attribute, pythonify) - if not (self.global_pythonify or pythonify) or 'errors' in updated_attribute: - return updated_attribute - a = MISPAttribute() - a.from_dict(**updated_attribute) - return a - - def delete_attribute(self, attribute: Union[MISPAttribute, int, str, UUID], hard: bool=False): - '''Delete an attribute from a MISP instance''' - attribute_id = self.__get_uuid_or_id_from_abstract_misp(attribute) - data = {} - if hard: - data['hard'] = 1 - response = self._prepare_request('POST', f'attributes/delete/{attribute_id}', data=data) - response = self._check_response(response, expect_json=True) - if ('errors' in response and response['errors'][0] == 403 - and response['errors'][1]['message'] == 'You do not have permission to do that.'): - # FIXME: https://github.com/MISP/MISP/issues/4913 - # At this point, we assume the user tried to delete an attribute on an event they don't own - # Re-try with a proposal - return self.delete_attribute_proposal(attribute_id) - return response - - # ## END Attribute ### - - # ## BEGIN Attribute Proposal ### - - def attribute_proposals(self, event: Union[MISPEvent, int, str, UUID]=None, pythonify: bool=False): - if event: - event_id = self.__get_uuid_or_id_from_abstract_misp(event) - attribute_proposals = self._prepare_request('GET', f'shadow_attributes/index/{event_id}') - else: - attribute_proposals = self._prepare_request('GET', f'shadow_attributes') - attribute_proposals = self._check_response(attribute_proposals, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in attribute_proposals: - return attribute_proposals - to_return = [] - for attribute_proposal in attribute_proposals: - a = MISPShadowAttribute() - a.from_dict(**attribute_proposal) - to_return.append(a) - return to_return - - def get_attribute_proposal(self, proposal: Union[MISPShadowAttribute, int, str, UUID], pythonify: bool=False): - proposal_id = self.__get_uuid_or_id_from_abstract_misp(proposal) - attribute_proposal = self._prepare_request('GET', f'shadow_attributes/view/{proposal_id}') - attribute_proposal = self._check_response(attribute_proposal, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in attribute_proposal: - return attribute_proposal - a = MISPShadowAttribute() - a.from_dict(**attribute_proposal) - return a - - # NOTE: the tree following method have a very specific meaning, look at the comments - - def add_attribute_proposal(self, event: Union[MISPEvent, int, str, UUID], attribute: MISPAttribute, pythonify: bool=False): - '''Propose a new attribute in an event''' - event_id = self.__get_uuid_or_id_from_abstract_misp(event) - new_attribute_proposal = self._prepare_request('POST', f'shadow_attributes/add/{event_id}', data=attribute) - new_attribute_proposal = self._check_response(new_attribute_proposal, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in new_attribute_proposal: - return new_attribute_proposal - a = MISPShadowAttribute() - a.from_dict(**new_attribute_proposal) - return a - - def update_attribute_proposal(self, initial_attribute: Union[MISPAttribute, int, str, UUID], attribute: MISPAttribute, pythonify: bool=False): - '''Propose a change for an attribute''' - initial_attribute_id = self.__get_uuid_or_id_from_abstract_misp(initial_attribute) - if self._old_misp((2, 4, 112), '2020-01-01', sys._getframe().f_code.co_name): - # Inconsistency in MISP: https://github.com/MISP/MISP/issues/4857 - # Fix: https://github.com/MISP/MISP/commit/d6a15438f7a53f589ddeabe2b14e65c92baf43d3 - attribute = {'ShadowAttribute': attribute} - update_attribute_proposal = self._prepare_request('POST', f'shadow_attributes/edit/{initial_attribute_id}', data=attribute) - update_attribute_proposal = self._check_response(update_attribute_proposal, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in update_attribute_proposal: - return update_attribute_proposal - a = MISPShadowAttribute() - a.from_dict(**update_attribute_proposal) - return a - - def delete_attribute_proposal(self, attribute: Union[MISPAttribute, int, str, UUID]): - '''Propose the deletion of an attribute''' - attribute_id = self.__get_uuid_or_id_from_abstract_misp(attribute) - response = self._prepare_request('POST', f'shadow_attributes/delete/{attribute_id}') - return self._check_response(response, expect_json=True) - - # NOTE: You cannot modify an existing proposal, only accept/discard - - def accept_attribute_proposal(self, proposal: Union[MISPShadowAttribute, int, str, UUID]): - '''Accept a proposal''' - proposal_id = self.__get_uuid_or_id_from_abstract_misp(proposal) - response = self._prepare_request('POST', f'shadow_attributes/accept/{proposal_id}') - return self._check_response(response, expect_json=True) - - def discard_attribute_proposal(self, proposal: Union[MISPShadowAttribute, int, str, UUID]): - '''Discard a proposal''' - proposal_id = self.__get_uuid_or_id_from_abstract_misp(proposal) - response = self._prepare_request('POST', f'shadow_attributes/discard/{proposal_id}') - return self._check_response(response, expect_json=True) - - # ## END Attribute Proposal ### - - # ## BEGIN Sighting ### - - def sightings(self, misp_entity: AbstractMISP=None, org: Union[MISPOrganisation, int, str, UUID]=None, pythonify: bool=False): - """Get the list of sighting related to a MISPEvent or a MISPAttribute (depending on type of misp_entity)""" - if isinstance(misp_entity, MISPEvent): - context = 'event' - elif isinstance(misp_entity, MISPAttribute): - context = 'attribute' - else: - context = None - if org is not None: - org_id = self.__get_uuid_or_id_from_abstract_misp(org) - else: - org_id = None - - if self._old_misp((2, 4, 112), '2020-01-01', sys._getframe().f_code.co_name): - url = f'sightings/listSightings/{misp_entity.id}/{context}' - if org_id: - url = f'{url}/{org_id}' - sightings = self._prepare_request('POST', url) - else: - if context is None: - url = 'sightings' - to_post = {} - else: - url = 'sightings/listSightings' - to_post = {'id': misp_entity.id, 'context': context} - if org_id: - to_post['org_id'] = org_id - sightings = self._prepare_request('POST', url, data=to_post) - - sightings = self._check_response(sightings, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in sightings: - return sightings - to_return = [] - for sighting in sightings: - s = MISPSighting() - s.from_dict(**sighting) - to_return.append(s) - return to_return - - def add_sighting(self, sighting: MISPSighting, attribute: Union[MISPAttribute, int, str, UUID]=None, pythonify: bool=False): - '''Add a new sighting (globally, or to a specific attribute)''' - if attribute: - attribute_id = self.__get_uuid_or_id_from_abstract_misp(attribute) - new_sighting = self._prepare_request('POST', f'sightings/add/{attribute_id}', data=sighting) - else: - # Either the ID/UUID is in the sighting, or we want to add a sighting on all the attributes with the given value - new_sighting = self._prepare_request('POST', f'sightings/add', data=sighting) - new_sighting = self._check_response(new_sighting, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in new_sighting: - return new_sighting - s = MISPSighting() - s.from_dict(**new_sighting) - return s - - def delete_sighting(self, sighting: Union[MISPSighting, int, str, UUID]): - '''Delete a sighting from a MISP instance''' - sighting_id = self.__get_uuid_or_id_from_abstract_misp(sighting) - response = self._prepare_request('POST', f'sightings/delete/{sighting_id}') - return self._check_response(response, expect_json=True) - - # ## END Sighting ### - - # ## BEGIN Tags ### - - def tags(self, pythonify: bool=False): - """Get the list of existing tags.""" - tags = self._prepare_request('GET', 'tags') - tags = self._check_response(tags, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in tags: - return tags['Tag'] - to_return = [] - for tag in tags['Tag']: - t = MISPTag() - t.from_dict(**tag) - to_return.append(t) - return to_return - - def get_tag(self, tag: Union[MISPTag, int, str, UUID], pythonify: bool=False): - """Get a tag by id.""" - tag_id = self.__get_uuid_or_id_from_abstract_misp(tag) - tag = self._prepare_request('GET', f'tags/view/{tag_id}') - tag = self._check_response(tag, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in tag: - return tag - t = MISPTag() - t.from_dict(**tag) - return t - - def add_tag(self, tag: MISPTag, pythonify: bool=False): - '''Add a new tag on a MISP instance - Notes: - * The user calling this method needs the Tag Editor permission - * It doesn't add a tag to an event, simply create it on a MISP instance. - ''' - new_tag = self._prepare_request('POST', 'tags/add', data=tag) - new_tag = self._check_response(new_tag, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in new_tag: - return new_tag - t = MISPTag() - t.from_dict(**new_tag) - return t - - def enable_tag(self, tag: MISPTag, pythonify: bool=False): - """Enable a tag.""" - tag.hide_tag = False - return self.update_tag(tag, pythonify=pythonify) - - def disable_tag(self, tag: MISPTag, pythonify: bool=False): - """Disable a tag.""" - tag.hide_tag = True - return self.update_tag(tag, pythonify=pythonify) - - def update_tag(self, tag: MISPTag, tag_id: int=None, pythonify: bool=False): - """Edit only the provided parameters of a tag.""" - if tag_id is None: - tag_id = self.__get_uuid_or_id_from_abstract_misp(tag) - else: - tag_id = self.__get_uuid_or_id_from_abstract_misp(tag_id) - if self._old_misp((2, 4, 114), '2020-01-01', sys._getframe().f_code.co_name): - # Inconsistency https://github.com/MISP/MISP/issues/4852 - tag = {'Tag': tag} - updated_tag = self._prepare_request('POST', f'tags/edit/{tag_id}', data=tag) - updated_tag = self._check_response(updated_tag, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in updated_tag: - return updated_tag - t = MISPTag() - t.from_dict(**updated_tag) - return t - - def delete_tag(self, tag: Union[MISPTag, int, str, UUID]): - '''Delete an attribute from a MISP instance''' - tag_id = self.__get_uuid_or_id_from_abstract_misp(tag) - response = self._prepare_request('POST', f'tags/delete/{tag_id}') - return self._check_response(response, expect_json=True) - - # ## END Tags ### - - # ## BEGIN Taxonomies ### - - def taxonomies(self, pythonify: bool=False): - """Get all the taxonomies.""" - taxonomies = self._prepare_request('GET', 'taxonomies') - taxonomies = self._check_response(taxonomies, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in taxonomies: - return taxonomies - to_return = [] - for taxonomy in taxonomies: - t = MISPTaxonomy() - t.from_dict(**taxonomy) - to_return.append(t) - return to_return - - def get_taxonomy(self, taxonomy: Union[MISPTaxonomy, int, str, UUID], pythonify: bool=False): - """Get a taxonomy from a MISP instance.""" - taxonomy_id = self.__get_uuid_or_id_from_abstract_misp(taxonomy) - taxonomy = self._prepare_request('GET', f'taxonomies/view/{taxonomy_id}') - taxonomy = self._check_response(taxonomy, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in taxonomy: - return taxonomy - t = MISPTaxonomy() - t.from_dict(**taxonomy) - return t - - def enable_taxonomy(self, taxonomy: Union[MISPTaxonomy, int, str, UUID]): - """Enable a taxonomy.""" - taxonomy_id = self.__get_uuid_or_id_from_abstract_misp(taxonomy) - response = self._prepare_request('POST', f'taxonomies/enable/{taxonomy_id}') - return self._check_response(response, expect_json=True) - - def disable_taxonomy(self, taxonomy: Union[MISPTaxonomy, int, str, UUID]): - """Disable a taxonomy.""" - taxonomy_id = self.__get_uuid_or_id_from_abstract_misp(taxonomy) - self.disable_taxonomy_tags(taxonomy_id) - response = self._prepare_request('POST', f'taxonomies/disable/{taxonomy_id}') - return self._check_response(response, expect_json=True) - - def disable_taxonomy_tags(self, taxonomy: Union[MISPTaxonomy, int, str, UUID]): - """Disable all the tags of a taxonomy.""" - taxonomy_id = self.__get_uuid_or_id_from_abstract_misp(taxonomy) - response = self._prepare_request('POST', f'taxonomies/disableTag/{taxonomy_id}') - return self._check_response(response, expect_json=True) - - def enable_taxonomy_tags(self, taxonomy: Union[MISPTaxonomy, int, str, UUID]): - """Enable all the tags of a taxonomy. - NOTE: this automatically done when you call enable_taxonomy.""" - taxonomy_id = self.__get_uuid_or_id_from_abstract_misp(taxonomy) - taxonomy = self.get_taxonomy(taxonomy_id) - if not taxonomy['Taxonomy']['enabled']: - raise PyMISPError(f"The taxonomy {taxonomy['Taxonomy']['name']} is not enabled.") - url = urljoin(self.root_url, 'taxonomies/addTag/{}'.format(taxonomy_id)) - response = self._prepare_request('POST', url) - return self._check_response(response, expect_json=True) - - def update_taxonomies(self): - """Update all the taxonomies.""" - response = self._prepare_request('POST', 'taxonomies/update') - return self._check_response(response, expect_json=True) - - # ## END Taxonomies ### - - # ## BEGIN Warninglists ### - - def warninglists(self, pythonify: bool=False): - """Get all the warninglists.""" - warninglists = self._prepare_request('GET', 'warninglists') - warninglists = self._check_response(warninglists, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in warninglists: - return warninglists['Warninglists'] - to_return = [] - for warninglist in warninglists['Warninglists']: - w = MISPWarninglist() - w.from_dict(**warninglist) - to_return.append(w) - return to_return - - def get_warninglist(self, warninglist: Union[MISPWarninglist, int, str, UUID], pythonify: bool=False): - """Get a warninglist.""" - warninglist_id = self.__get_uuid_or_id_from_abstract_misp(warninglist) - warninglist = self._prepare_request('GET', f'warninglists/view/{warninglist_id}') - warninglist = self._check_response(warninglist, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in warninglist: - return warninglist - w = MISPWarninglist() - w.from_dict(**warninglist) - return w - - def toggle_warninglist(self, warninglist_id: List[int]=None, warninglist_name: List[str]=None, - force_enable: bool=False): - '''Toggle (enable/disable) the status of a warninglist by ID. - :param warninglist_id: ID of the WarningList - :param force_enable: Force the warning list in the enabled state (does nothing is already enabled) - ''' - if warninglist_id is None and warninglist_name is None: - raise PyMISPError('Either warninglist_id or warninglist_name is required.') - query = {} - if warninglist_id is not None: - if not isinstance(warninglist_id, list): - warninglist_id = [warninglist_id] - query['id'] = warninglist_id - if warninglist_name is not None: - if not isinstance(warninglist_name, list): - warninglist_name = [warninglist_name] - query['name'] = warninglist_name - if force_enable: - query['enabled'] = force_enable - response = self._prepare_request('POST', 'warninglists/toggleEnable', data=json.dumps(query)) - return self._check_response(response, expect_json=True) - - def enable_warninglist(self, warninglist: Union[MISPWarninglist, int, str, UUID]): - """Enable a warninglist.""" - warninglist_id = self.__get_uuid_or_id_from_abstract_misp(warninglist) - return self.toggle_warninglist(warninglist_id=warninglist_id, force_enable=True) - - def disable_warninglist(self, warninglist: Union[MISPWarninglist, int, str, UUID]): - """Disable a warninglist.""" - warninglist_id = self.__get_uuid_or_id_from_abstract_misp(warninglist) - return self.toggle_warninglist(warninglist_id=warninglist_id, force_enable=False) - - def values_in_warninglist(self, value: list): - """Check if IOC values are in warninglist""" - response = self._prepare_request('POST', 'warninglists/checkValue', data=json.dumps(value)) - return self._check_response(response, expect_json=True) - - def update_warninglists(self): - """Update all the warninglists.""" - response = self._prepare_request('POST', 'warninglists/update') - return self._check_response(response, expect_json=True) - - # ## END Warninglists ### - - # ## BEGIN Noticelist ### - - def noticelists(self, pythonify: bool=False): - """Get all the noticelists.""" - noticelists = self._prepare_request('GET', 'noticelists') - noticelists = self._check_response(noticelists, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in noticelists: - return noticelists - to_return = [] - for noticelist in noticelists: - n = MISPNoticelist() - n.from_dict(**noticelist) - to_return.append(n) - return to_return - - def get_noticelist(self, noticelist: Union[MISPNoticelist, int, str, UUID], pythonify: bool=False): - """Get a noticelist by id.""" - noticelist_id = self.__get_uuid_or_id_from_abstract_misp(noticelist) - noticelist = self._prepare_request('GET', f'noticelists/view/{noticelist_id}') - noticelist = self._check_response(noticelist, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in noticelist: - return noticelist - n = MISPNoticelist() - n.from_dict(**noticelist) - return n - - def enable_noticelist(self, noticelist: Union[MISPNoticelist, int, str, UUID]): - """Enable a noticelist by id.""" - # FIXME: https://github.com/MISP/MISP/issues/4856 - # response = self._prepare_request('POST', f'noticelists/enable/{noticelist_id}') - noticelist_id = self.__get_uuid_or_id_from_abstract_misp(noticelist) - response = self._prepare_request('POST', f'noticelists/enableNoticelist/{noticelist_id}/true') - return self._check_response(response, expect_json=True) - - def disable_noticelist(self, noticelist: Union[MISPNoticelist, int, str, UUID]): - """Disable a noticelist by id.""" - # FIXME: https://github.com/MISP/MISP/issues/4856 - # response = self._prepare_request('POST', f'noticelists/disable/{noticelist_id}') - noticelist_id = self.__get_uuid_or_id_from_abstract_misp(noticelist) - response = self._prepare_request('POST', f'noticelists/enableNoticelist/{noticelist_id}') - return self._check_response(response, expect_json=True) - - def update_noticelists(self): - """Update all the noticelists.""" - response = self._prepare_request('POST', 'noticelists/update') - return self._check_response(response, expect_json=True) - - # ## END Noticelist ### - - # ## BEGIN Galaxy ### - - def galaxies(self, pythonify: bool=False): - """Get all the galaxies.""" - galaxies = self._prepare_request('GET', 'galaxies') - galaxies = self._check_response(galaxies, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in galaxies: - return galaxies - to_return = [] - for galaxy in galaxies: - g = MISPGalaxy() - g.from_dict(**galaxy) - to_return.append(g) - return to_return - - def get_galaxy(self, galaxy: Union[MISPGalaxy, int, str, UUID], pythonify: bool=False): - """Get a galaxy by id.""" - galaxy_id = self.__get_uuid_or_id_from_abstract_misp(galaxy) - galaxy = self._prepare_request('GET', f'galaxies/view/{galaxy_id}') - galaxy = self._check_response(galaxy, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in galaxy: - return galaxy - g = MISPGalaxy() - g.from_dict(**galaxy) - return g - - def update_galaxies(self): - """Update all the galaxies.""" - response = self._prepare_request('POST', 'galaxies/update') - return self._check_response(response, expect_json=True) - - # ## END Galaxy ### - - # ## BEGIN Feed ### - - def feeds(self, pythonify: bool=False): - """Get the list of existing feeds.""" - feeds = self._prepare_request('GET', 'feeds') - feeds = self._check_response(feeds, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in feeds: - return feeds - to_return = [] - for feed in feeds: - f = MISPFeed() - f.from_dict(**feed) - to_return.append(f) - return to_return - - def get_feed(self, feed: Union[MISPFeed, int, str, UUID], pythonify: bool=False): - """Get a feed by id.""" - feed_id = self.__get_uuid_or_id_from_abstract_misp(feed) - feed = self._prepare_request('GET', f'feeds/view/{feed_id}') - feed = self._check_response(feed, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in feed: - return feed - f = MISPFeed() - f.from_dict(**feed) - return f - - def add_feed(self, feed: MISPFeed, pythonify: bool=False): - '''Add a new feed on a MISP instance''' - # FIXME: https://github.com/MISP/MISP/issues/4834 - feed = {'Feed': feed} - new_feed = self._prepare_request('POST', 'feeds/add', data=feed) - new_feed = self._check_response(new_feed, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in new_feed: - return new_feed - f = MISPFeed() - f.from_dict(**new_feed) - return f - - def enable_feed(self, feed: Union[MISPFeed, int, str, UUID], pythonify: bool=False): - '''Enable a feed (fetching it will create event(s)''' - if not isinstance(feed, MISPFeed): - feed_id = self.__get_uuid_or_id_from_abstract_misp(feed) # In case we have a UUID - feed = MISPFeed() - feed.id = feed_id - feed.enabled = True - return self.update_feed(feed=feed, pythonify=pythonify) - - def disable_feed(self, feed: Union[MISPFeed, int, str, UUID], pythonify: bool=False): - '''Disable a feed''' - if not isinstance(feed, MISPFeed): - feed_id = self.__get_uuid_or_id_from_abstract_misp(feed) # In case we have a UUID - feed = MISPFeed() - feed.id = feed_id - feed.enabled = False - return self.update_feed(feed=feed, pythonify=pythonify) - - def enable_feed_cache(self, feed: Union[MISPFeed, int, str, UUID], pythonify: bool=False): - '''Enable the caching of a feed''' - if not isinstance(feed, MISPFeed): - feed_id = self.__get_uuid_or_id_from_abstract_misp(feed) # In case we have a UUID - feed = MISPFeed() - feed.id = feed_id - feed.caching_enabled = True - return self.update_feed(feed=feed, pythonify=pythonify) - - def disable_feed_cache(self, feed: Union[MISPFeed, int, str, UUID], pythonify: bool=False): - '''Disable the caching of a feed''' - if not isinstance(feed, MISPFeed): - feed_id = self.__get_uuid_or_id_from_abstract_misp(feed) # In case we have a UUID - feed = MISPFeed() - feed.id = feed_id - feed.caching_enabled = False - return self.update_feed(feed=feed, pythonify=pythonify) - - def update_feed(self, feed: MISPFeed, feed_id: int=None, pythonify: bool=False): - '''Update a feed on a MISP instance''' - if feed_id is None: - feed_id = self.__get_uuid_or_id_from_abstract_misp(feed) - else: - feed_id = self.__get_uuid_or_id_from_abstract_misp(feed_id) - # FIXME: https://github.com/MISP/MISP/issues/4834 - feed = {'Feed': feed} - updated_feed = self._prepare_request('POST', f'feeds/edit/{feed_id}', data=feed) - updated_feed = self._check_response(updated_feed, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in updated_feed: - return updated_feed - f = MISPFeed() - f.from_dict(**updated_feed) - return f - - def delete_feed(self, feed: Union[MISPFeed, int, str, UUID]): - '''Delete a feed from a MISP instance''' - feed_id = self.__get_uuid_or_id_from_abstract_misp(feed) - response = self._prepare_request('POST', f'feeds/delete/{feed_id}') - return self._check_response(response, expect_json=True) - - def fetch_feed(self, feed: Union[MISPFeed, int, str, UUID]): - """Fetch one single feed""" - feed_id = self.__get_uuid_or_id_from_abstract_misp(feed) - response = self._prepare_request('GET', f'feeds/fetchFromFeed/{feed_id}') - return self._check_response(response) - - def cache_all_feeds(self): - """ Cache all the feeds""" - response = self._prepare_request('GET', 'feeds/cacheFeeds/all') - return self._check_response(response) - - def cache_feed(self, feed: Union[MISPFeed, int, str, UUID]): - """Cache a specific feed""" - feed_id = self.__get_uuid_or_id_from_abstract_misp(feed) - response = self._prepare_request('GET', f'feeds/cacheFeeds/{feed_id}') - return self._check_response(response) - - def cache_freetext_feeds(self): - """Cache all the freetext feeds""" - response = self._prepare_request('GET', 'feeds/cacheFeeds/freetext') - return self._check_response(response) - - def cache_misp_feeds(self): - """Cache all the MISP feeds""" - response = self._prepare_request('GET', 'feeds/cacheFeeds/misp') - return self._check_response(response) - - def compare_feeds(self): - """Generate the comparison matrix for all the MISP feeds""" - response = self._prepare_request('GET', 'feeds/compareFeeds') - return self._check_response(response) - - # ## END Feed ### - - # ## BEGIN Server ### - - def servers(self, pythonify: bool=False): - """Get the existing servers the MISP instance can synchronise with""" - servers = self._prepare_request('GET', 'servers') - servers = self._check_response(servers, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in servers: - return servers - to_return = [] - for server in servers: - s = MISPServer() - s.from_dict(**server) - to_return.append(s) - return to_return - - def get_sync_config(self, pythonify: bool=False): - '''WARNING: This method only works if the user calling it is a sync user''' - server = self._prepare_request('GET', 'servers/createSync') - server = self._check_response(server, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in server: - return server - s = MISPServer() - s.from_dict(**server) - return s - - def import_server(self, server: MISPServer, pythonify: bool=False): - """Import a sync server config received from get_sync_config""" - server = self._prepare_request('POST', f'servers/import', data=server) - server = self._check_response(server, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in server: - return server - s = MISPServer() - s.from_dict(**server) - return s - - def add_server(self, server: MISPServer, pythonify: bool=False): - """Add a server to synchronise with. - Note: You probably fant to use ExpandedPyMISP.get_sync_config and ExpandedPyMISP.import_server instead""" - server = self._prepare_request('POST', f'servers/add', data=server) - server = self._check_response(server, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in server: - return server - s = MISPServer() - s.from_dict(**server) - return s - - def update_server(self, server: MISPServer, server_id: int=None, pythonify: bool=False): - '''Update a server to synchronise with''' - if server_id is None: - server_id = self.__get_uuid_or_id_from_abstract_misp(server) - else: - server_id = self.__get_uuid_or_id_from_abstract_misp(server_id) - updated_server = self._prepare_request('POST', f'servers/edit/{server_id}', data=server) - updated_server = self._check_response(updated_server, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in updated_server: - return updated_server - s = MISPServer() - s.from_dict(**updated_server) - return s - - def delete_server(self, server: Union[MISPServer, int, str, UUID]): - '''Delete a sync server''' - server_id = self.__get_uuid_or_id_from_abstract_misp(server) - response = self._prepare_request('POST', f'servers/delete/{server_id}') - return self._check_response(response, expect_json=True) - - def server_pull(self, server: Union[MISPServer, int, str, UUID], event: Union[MISPEvent, int, str, UUID]=None): - '''Initialize a pull from a sync server''' - server_id = self.__get_uuid_or_id_from_abstract_misp(server) - if event: - event_id = self.__get_uuid_or_id_from_abstract_misp(event) - url = f'servers/pull/{server_id}/{event_id}' - else: - url = f'servers/pull/{server_id}' - response = self._prepare_request('GET', url) - # FIXME: can we pythonify? - return self._check_response(response) - - def server_push(self, server: Union[MISPServer, int, str, UUID], event: Union[MISPEvent, int, str, UUID]=None): - '''Initialize a push to a sync server''' - server_id = self.__get_uuid_or_id_from_abstract_misp(server) - if event: - event_id = self.__get_uuid_or_id_from_abstract_misp(event) - url = f'servers/push/{server_id}/{event_id}' - else: - url = f'servers/push/{server_id}' - response = self._prepare_request('GET', url) - # FIXME: can we pythonify? - return self._check_response(response) - - def test_server(self, server: Union[MISPServer, int, str, UUID]): - server_id = self.__get_uuid_or_id_from_abstract_misp(server) - response = self._prepare_request('POST', f'servers/testConnection/{server_id}') - return self._check_response(response, expect_json=True) - - # ## END Server ### - - # ## BEGIN Sharing group ### - - def sharing_groups(self, pythonify: bool=False): - """Get the existing sharing groups""" - sharing_groups = self._prepare_request('GET', 'sharing_groups') - sharing_groups = self._check_response(sharing_groups, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in sharing_groups: - return sharing_groups - to_return = [] - for sharing_group in sharing_groups: - s = MISPSharingGroup() - s.from_dict(**sharing_group) - to_return.append(s) - return to_return - - def add_sharing_group(self, sharing_group: MISPSharingGroup, pythonify: bool=False): - """Add a new sharing group""" - sharing_group = self._prepare_request('POST', f'sharing_groups/add', data=sharing_group) - sharing_group = self._check_response(sharing_group, expect_json=True) - if self._old_misp((2, 4, 112), '2020-01-01', sys._getframe().f_code.co_name) and isinstance(sharing_group, list): - # https://github.com/MISP/MISP/issues/4882 - # https://github.com/MISP/MISP/commit/d75c6c9e3b7874fd0f083445126743873e5c53c4 - sharing_group = sharing_group[0] - if not (self.global_pythonify or pythonify) or 'errors' in sharing_group: - return sharing_group - s = MISPSharingGroup() - s.from_dict(**sharing_group) - return s - - def delete_sharing_group(self, sharing_group: Union[MISPSharingGroup, int, str, UUID]): - """Delete a sharing group""" - sharing_group_id = self.__get_uuid_or_id_from_abstract_misp(sharing_group) - response = self._prepare_request('POST', f'sharing_groups/delete/{sharing_group_id}') - return self._check_response(response, expect_json=True) - - def add_org_to_sharing_group(self, sharing_group: Union[MISPSharingGroup, int, str, UUID], - organisation: Union[MISPOrganisation, int, str, UUID], extend: bool=False): - '''Add an organisation to a sharing group. - :sharing_group: Sharing group's local instance ID, or Sharing group's global UUID - :organisation: Organisation's local instance ID, or Organisation's global UUID, or Organisation's name as known to the curent instance - :extend: Allow the organisation to extend the group - ''' - sharing_group_id = self.__get_uuid_or_id_from_abstract_misp(sharing_group) - organisation_id = self.__get_uuid_or_id_from_abstract_misp(organisation) - to_jsonify = {'sg_id': sharing_group_id, 'org_id': organisation_id, 'extend': extend} - response = self._prepare_request('POST', 'sharingGroups/addOrg', data=to_jsonify) - return self._check_response(response) - - def remove_org_from_sharing_group(self, sharing_group: Union[MISPSharingGroup, int, str, UUID], - organisation: Union[MISPOrganisation, int, str, UUID]): - '''Remove an organisation from a sharing group. - :sharing_group: Sharing group's local instance ID, or Sharing group's global UUID - :organisation: Organisation's local instance ID, or Organisation's global UUID, or Organisation's name as known to the curent instance - ''' - sharing_group_id = self.__get_uuid_or_id_from_abstract_misp(sharing_group) - organisation_id = self.__get_uuid_or_id_from_abstract_misp(organisation) - to_jsonify = {'sg_id': sharing_group_id, 'org_id': organisation_id} - response = self._prepare_request('POST', 'sharingGroups/removeOrg', data=to_jsonify) - return self._check_response(response) - - def add_server_to_sharing_group(self, sharing_group: Union[MISPSharingGroup, int, str, UUID], - server: Union[MISPServer, int, str, UUID], all_orgs: bool=False): - '''Add a server to a sharing group. - :sharing_group: Sharing group's local instance ID, or Sharing group's global UUID - :server: Server's local instance ID, or URL of the Server, or Server's name as known to the curent instance - :all_orgs: Add all the organisations of the server to the group - ''' - sharing_group_id = self.__get_uuid_or_id_from_abstract_misp(sharing_group) - server_id = self.__get_uuid_or_id_from_abstract_misp(server) - to_jsonify = {'sg_id': sharing_group_id, 'server_id': server_id, 'all_orgs': all_orgs} - response = self._prepare_request('POST', 'sharingGroups/addServer', data=to_jsonify) - return self._check_response(response) - - def remove_server_from_sharing_group(self, sharing_group: Union[MISPSharingGroup, int, str, UUID], - server: Union[MISPServer, int, str, UUID]): - '''Remove a server from a sharing group. - :sharing_group: Sharing group's local instance ID, or Sharing group's global UUID - :server: Server's local instance ID, or URL of the Server, or Server's name as known to the curent instance - ''' - sharing_group_id = self.__get_uuid_or_id_from_abstract_misp(sharing_group) - server_id = self.__get_uuid_or_id_from_abstract_misp(server) - to_jsonify = {'sg_id': sharing_group_id, 'server_id': server_id} - response = self._prepare_request('POST', 'sharingGroups/removeServer', data=to_jsonify) - return self._check_response(response) - - # ## END Sharing groups ### - - # ## BEGIN Organisation ### - - def organisations(self, scope="local", pythonify: bool=False): - """Get all the organisations.""" - organisations = self._prepare_request('GET', f'organisations/index/scope:{scope}') - organisations = self._check_response(organisations, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in organisations: - return organisations - to_return = [] - for organisation in organisations: - o = MISPOrganisation() - o.from_dict(**organisation) - to_return.append(o) - return to_return - - def get_organisation(self, organisation: Union[MISPOrganisation, int, str, UUID], pythonify: bool=False): - '''Get an organisation.''' - organisation_id = self.__get_uuid_or_id_from_abstract_misp(organisation) - organisation = self._prepare_request('GET', f'organisations/view/{organisation_id}') - organisation = self._check_response(organisation, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in organisation: - return organisation - o = MISPOrganisation() - o.from_dict(**organisation) - return o - - def add_organisation(self, organisation: MISPOrganisation, pythonify: bool=False): - '''Add an organisation''' - new_organisation = self._prepare_request('POST', f'admin/organisations/add', data=organisation) - new_organisation = self._check_response(new_organisation, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in new_organisation: - return new_organisation - o = MISPOrganisation() - o.from_dict(**new_organisation) - return o - - def update_organisation(self, organisation: MISPOrganisation, organisation_id: int=None, pythonify: bool=False): - '''Update an organisation''' - if organisation_id is None: - organisation_id = self.__get_uuid_or_id_from_abstract_misp(organisation) - else: - organisation_id = self.__get_uuid_or_id_from_abstract_misp(organisation_id) - updated_organisation = self._prepare_request('POST', f'admin/organisations/edit/{organisation_id}', data=organisation) - updated_organisation = self._check_response(updated_organisation, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in updated_organisation: - return updated_organisation - o = MISPOrganisation() - o.from_dict(**organisation) - return o - - def delete_organisation(self, organisation: Union[MISPOrganisation, int, str, UUID]): - '''Delete an organisation''' - # NOTE: MISP in inconsistent and currently require "delete" in the path and doesn't support HTTP DELETE - organisation_id = self.__get_uuid_or_id_from_abstract_misp(organisation) - response = self._prepare_request('POST', f'admin/organisations/delete/{organisation_id}') - return self._check_response(response, expect_json=True) - - # ## END Organisation ### - - # ## BEGIN User ### - - def users(self, pythonify: bool=False): - """Get all the users.""" - users = self._prepare_request('GET', 'admin/users') - users = self._check_response(users, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in users: - return users - to_return = [] - for user in users: - u = MISPUser() - u.from_dict(**user) - to_return.append(u) - return to_return - - def get_user(self, user: Union[MISPUser, int, str, UUID]='me', pythonify: bool=False, expanded: bool=False): - '''Get a user. `me` means the owner of the API key doing the query. - expanded also returns a MISPRole and a MISPUserSetting''' - user_id = self.__get_uuid_or_id_from_abstract_misp(user) - user = self._prepare_request('GET', f'users/view/{user_id}') - user = self._check_response(user, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in user: - return user - u = MISPUser() - u.from_dict(**user) - if not expanded: - return u - else: - if self._old_misp((2, 4, 117), '2020-01-01', sys._getframe().f_code.co_name): - return u, None, None - r = MISPRole() - r.from_dict(**user['Role']) - usersettings = [] - if user['UserSetting']: - for name, value in user['UserSetting'].items(): - us = MISPUserSetting() - us.from_dict(**{'name': name, 'value': value}) - usersettings.append(us) - return u, r, usersettings - - def add_user(self, user: MISPUser, pythonify: bool=False): - '''Add a new user''' - user = self._prepare_request('POST', f'admin/users/add', data=user) - user = self._check_response(user, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in user: - return user - u = MISPUser() - u.from_dict(**user) - return u - - def update_user(self, user: MISPUser, user_id: int=None, pythonify: bool=False): - '''Update an event on a MISP instance''' - if user_id is None: - user_id = self.__get_uuid_or_id_from_abstract_misp(user) - else: - user_id = self.__get_uuid_or_id_from_abstract_misp(user_id) - url = f'users/edit/{user_id}' - if self._current_role.perm_admin or self._current_role.perm_site_admin: - url = f'admin/{url}' - updated_user = self._prepare_request('POST', url, data=user) - updated_user = self._check_response(updated_user, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in updated_user: - return updated_user - e = MISPUser() - e.from_dict(**updated_user) - return e - - def delete_user(self, user: Union[MISPUser, int, str, UUID]): - '''Delete a user''' - # NOTE: MISP in inconsistent and currently require "delete" in the path and doesn't support HTTP DELETE - user_id = self.__get_uuid_or_id_from_abstract_misp(user) - response = self._prepare_request('POST', f'admin/users/delete/{user_id}') - return self._check_response(response, expect_json=True) - - def change_user_password(self, new_password: str, user: Union[MISPUser, int, str, UUID]=None): - response = self._prepare_request('POST', f'users/change_pw', data={'password': new_password}) - return self._check_response(response, expect_json=True) - - # ## END User ### - - # ## BEGIN Role ### - - def roles(self, pythonify: bool=False): - """Get the existing roles""" - roles = self._prepare_request('GET', 'roles') - roles = self._check_response(roles, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in roles: - return roles - to_return = [] - for role in roles: - r = MISPRole() - r.from_dict(**role) - to_return.append(r) - return to_return - - def set_default_role(self, role: Union[MISPRole, int, str, UUID]): - role_id = self.__get_uuid_or_id_from_abstract_misp(role) - url = urljoin(self.root_url, f'/admin/roles/set_default/{role_id}') - response = self._prepare_request('POST', url) - return self._check_response(response, expect_json=True) - - # ## END Role ### - - # ## BEGIN Search methods ### - - def search(self, controller: str='events', return_format: str='json', - limit: Optional[int]=None, page: Optional[int]=None, - value: Optional[SearchParameterTypes]=None, - type_attribute: Optional[SearchParameterTypes]=None, - category: Optional[SearchParameterTypes]=None, - org: Optional[SearchParameterTypes]=None, - tags: Optional[SearchParameterTypes]=None, - quick_filter: Optional[str]=None, quickFilter: Optional[str]=None, - date_from: Optional[DateTypes]=None, - date_to: Optional[DateTypes]=None, - eventid: Optional[SearchType]=None, - with_attachments: Optional[bool]=None, withAttachments: Optional[bool]=None, - metadata: Optional[bool]=None, - uuid: Optional[str]=None, - publish_timestamp: Optional[DateInterval]=None, last: Optional[DateInterval]=None, - timestamp: Optional[DateInterval]=None, - published: Optional[bool]=None, - enforce_warninglist: Optional[bool]=None, enforceWarninglist: Optional[bool]=None, - to_ids: Optional[Union[ToIDSType, List[ToIDSType]]]=None, - deleted: Optional[str]=None, - include_event_uuid: Optional[bool]=None, includeEventUuid: Optional[bool]=None, - include_event_tags: Optional[bool]=None, includeEventTags: Optional[bool]=None, - event_timestamp: Optional[DateTypes]=None, - sg_reference_only: Optional[bool]=None, - eventinfo: Optional[str]=None, - searchall: Optional[bool]=None, - requested_attributes: Optional[str]=None, - include_context: Optional[bool]=None, includeContext: Optional[bool]=None, - headerless: Optional[bool]=None, - include_sightings: Optional[bool]=None, includeSightings: Optional[bool]=None, - include_correlations: Optional[bool]=None, includeCorrelations: Optional[bool]=None, - pythonify: Optional[bool]=False, - **kwargs): - '''Search in the MISP instance - - :param return_format: Set the return format of the search (Currently supported: json, xml, openioc, suricata, snort - more formats are being moved to restSearch with the goal being that all searches happen through this API). Can be passed as the first parameter after restSearch or via the JSON payload. - :param limit: Limit the number of results returned, depending on the scope (for example 10 attributes or 10 full events). - :param page: If a limit is set, sets the page to be returned. page 3, limit 100 will return records 201->300). - :param value: Search for the given value in the attributes' value field. - :param type_attribute: The attribute type, any valid MISP attribute type is accepted. - :param category: The attribute category, any valid MISP attribute category is accepted. - :param org: Search by the creator organisation by supplying the organisation identifier. - :param tags: Tags to search or to exclude. You can pass a list, or the output of `build_complex_query` - :param quick_filter: The string passed to this field will ignore all of the other arguments. MISP will return an xml / json (depending on the header sent) of all events that have a sub-string match on value in the event info, event orgc, or any of the attribute value1 / value2 fields, or in the attribute comment. - :param date_from: Events with the date set to a date after the one specified. This filter will use the date of the event. - :param date_to: Events with the date set to a date before the one specified. This filter will use the date of the event. - :param eventid: The events that should be included / excluded from the search - :param with_attachments: If set, encodes the attachments / zipped malware samples as base64 in the data field within each attribute - :param metadata: Only the metadata (event, tags, relations) is returned, attributes and proposals are omitted. - :param uuid: Restrict the results by uuid. - :param publish_timestamp: Restrict the results by the last publish timestamp (newer than). - :param timestamp: Restrict the results by the timestamp (last edit). Any event with a timestamp newer than the given timestamp will be returned. In case you are dealing with /attributes as scope, the attribute's timestamp will be used for the lookup. - :param published: Set whether published or unpublished events should be returned. Do not set the parameter if you want both. - :param enforce_warninglist: Remove any attributes from the result that would cause a hit on a warninglist entry. - :param to_ids: By default all attributes are returned that match the other filter parameters, irregardless of their to_ids setting. To restrict the returned data set to to_ids only attributes set this parameter to 1. 0 for the ones with to_ids set to False. - :param deleted: If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using "only" as a parameter it will limit the returned data set to soft-deleted data only. - :param include_event_uuid: Instead of just including the event ID, also include the event UUID in each of the attributes. - :param include_event_tags: Include the event level tags in each of the attributes. - :param event_timestamp: Only return attributes from events that have received a modification after the given timestamp. - :param sg_reference_only: If this flag is set, sharing group objects will not be included, instead only the sharing group ID is set. - :param eventinfo: Filter on the event's info field. - :param searchall: Search for a full or a substring (delimited by % for substrings) in the event info, event tags, attribute tags, attribute values or attribute comment fields. - :param requested_attributes: [CSV only] Select the fields that you wish to include in the CSV export. By setting event level fields additionally, includeContext is not required to get event metadata. - :param include_context: [Attribute only] Include the event data with each attribute. - :param headerless: [CSV Only] The CSV created when this setting is set to true will not contain the header row. - :param include_sightings: [JSON Only - Attribute] Include the sightings of the matching attributes. - :param include_correlations: [JSON Only - attribute] Include the correlations of the matching attributes. - :param pythonify: Returns a list of PyMISP Objects instead of the plain json output. Warning: it might use a lot of RAM - - Deprecated: - - :param quickFilter: synponym for quick_filter - :param withAttachments: synonym for with_attachments - :param last: synonym for publish_timestamp - :param enforceWarninglist: synonym for enforce_warninglist - :param includeEventUuid: synonym for include_event_uuid - :param includeEventTags: synonym for include_event_tags - :param includeContext: synonym for include_context - - ''' - - return_formats = ['openioc', 'json', 'xml', 'suricata', 'snort', 'text', 'rpz', 'csv', 'cache', 'stix', 'stix2', 'yara', 'yara-json', 'attack', 'attack-sightings'] - - if controller not in ['events', 'attributes', 'objects', 'sightings']: - raise ValueError('controller has to be in {}'.format(', '.join(['events', 'attributes', 'objects']))) - - # Deprecated stuff / synonyms - if quickFilter is not None: - quick_filter = quickFilter - if withAttachments is not None: - with_attachments = withAttachments - if last is not None: - publish_timestamp = last - if enforceWarninglist is not None: - enforce_warninglist = enforceWarninglist - if includeEventUuid is not None: - include_event_uuid = includeEventUuid - if includeEventTags is not None: - include_event_tags = includeEventTags - if includeContext is not None: - include_context = includeContext - if includeCorrelations is not None: - include_correlations = includeCorrelations - if includeSightings is not None: - include_sightings = includeSightings - # Add all the parameters in kwargs are aimed at modules, or other 3rd party components, and cannot be sanitized. - # They are passed as-is. - query = kwargs - - if return_format not in return_formats: - raise ValueError('return_format has to be in {}'.format(', '.join(return_formats))) - query['returnFormat'] = return_format - - query['page'] = page - query['limit'] = limit - query['value'] = value - query['type'] = type_attribute - query['category'] = category - query['org'] = org - query['tags'] = tags - query['quickFilter'] = quick_filter - query['from'] = self._make_timestamp(date_from) - query['to'] = self._make_timestamp(date_to) - query['eventid'] = eventid - query['withAttachments'] = self._make_misp_bool(with_attachments) - query['metadata'] = self._make_misp_bool(metadata) - query['uuid'] = uuid - if publish_timestamp is not None: - if isinstance(publish_timestamp, (list, tuple)): - query['publish_timestamp'] = (self._make_timestamp(publish_timestamp[0]), self._make_timestamp(publish_timestamp[1])) - else: - query['publish_timestamp'] = self._make_timestamp(publish_timestamp) - if timestamp is not None: - if isinstance(timestamp, (list, tuple)): - query['timestamp'] = (self._make_timestamp(timestamp[0]), self._make_timestamp(timestamp[1])) - else: - query['timestamp'] = self._make_timestamp(timestamp) - query['published'] = published - query['enforceWarninglist'] = self._make_misp_bool(enforce_warninglist) - if to_ids is not None: - if int(to_ids) not in [0, 1]: - raise ValueError('to_ids has to be in {}'.format(', '.join([0, 1]))) - query['to_ids'] = to_ids - query['deleted'] = deleted - query['includeEventUuid'] = self._make_misp_bool(include_event_uuid) - query['includeEventTags'] = self._make_misp_bool(include_event_tags) - if event_timestamp is not None: - if isinstance(event_timestamp, (list, tuple)): - query['event_timestamp'] = (self._make_timestamp(event_timestamp[0]), self._make_timestamp(event_timestamp[1])) - else: - query['event_timestamp'] = self._make_timestamp(event_timestamp) - query['sgReferenceOnly'] = self._make_misp_bool(sg_reference_only) - query['eventinfo'] = eventinfo - query['searchall'] = searchall - query['requested_attributes'] = requested_attributes - query['includeContext'] = self._make_misp_bool(include_context) - query['headerless'] = self._make_misp_bool(headerless) - query['includeSightings'] = self._make_misp_bool(include_sightings) - query['includeCorrelations'] = self._make_misp_bool(include_correlations) - url = urljoin(self.root_url, f'{controller}/restSearch') - response = self._prepare_request('POST', url, data=query) - if return_format == 'json': - normalized_response = self._check_response(response, expect_json=True) - else: - normalized_response = self._check_response(response) - - if return_format == 'csv' and (self.global_pythonify or pythonify) and not headerless: - return self._csv_to_dict(normalized_response) - - if 'errors' in normalized_response: - return normalized_response - - if return_format == 'json' and self.global_pythonify or pythonify: - # The response is in json, we can convert it to a list of pythonic MISP objects - to_return = [] - if controller == 'events': - for e in normalized_response: - me = MISPEvent() - me.load(e) - to_return.append(me) - elif controller == 'attributes': - # FIXME: obvs, this is hurting my soul. We need something generic. - for a in normalized_response.get('Attribute'): - ma = MISPAttribute() - ma.from_dict(**a) - if 'Event' in ma: - me = MISPEvent() - me.from_dict(**ma.Event) - ma.Event = me - if 'RelatedAttribute' in ma: - related_attributes = [] - for ra in ma.RelatedAttribute: - r_attribute = MISPAttribute() - r_attribute.from_dict(**ra) - if 'Event' in r_attribute: - me = MISPEvent() - me.from_dict(**r_attribute.Event) - r_attribute.Event = me - related_attributes.append(r_attribute) - ma.RelatedAttribute = related_attributes - if 'Sighting' in ma: - sightings = [] - for sighting in ma.Sighting: - s = MISPSighting() - s.from_dict(**sighting) - sightings.append(s) - ma.Sighting = sightings - to_return.append(ma) - elif controller == 'objects': - raise PyMISPNotImplementedYet('Not implemented yet') - return to_return - - return normalized_response - - def search_index(self, published: Optional[bool]=None, eventid: Optional[SearchType]=None, - tags: Optional[SearchParameterTypes]=None, - date_from: Optional[DateTypes]=None, - date_to: Optional[DateTypes]=None, - eventinfo: Optional[str]=None, - threatlevel: Optional[List[SearchType]]=None, - distribution: Optional[List[SearchType]]=None, - analysis: Optional[List[SearchType]]=None, - org: Optional[SearchParameterTypes]=None, - timestamp: Optional[DateInterval]=None, - pythonify: Optional[bool]=None): - """Search only at the index level. Using ! in front of a value means NOT (default is OR) - - :param published: Set whether published or unpublished events should be returned. Do not set the parameter if you want both. - :param eventid: The events that should be included / excluded from the search - :param tags: Tags to search or to exclude. You can pass a list, or the output of `build_complex_query` - :param date_from: Events with the date set to a date after the one specified. This filter will use the date of the event. - :param date_to: Events with the date set to a date before the one specified. This filter will use the date of the event. - :param eventinfo: Filter on the event's info field. - :param threatlevel: Threat level(s) (1,2,3,4) | list - :param distribution: Distribution level(s) (0,1,2,3) | list - :param analysis: Analysis level(s) (0,1,2) | list - :param org: Search by the creator organisation by supplying the organisation identifier. - :param timestamp: Restrict the results by the timestamp (last edit). Any event with a timestamp newer than the given timestamp will be returned. In case you are dealing with /attributes as scope, the attribute's timestamp will be used for the lookup. - :param pythonify: Returns a list of PyMISP Objects instead or the plain json output. Warning: it might use a lot of RAM - """ - query = locals() - query.pop('self') - query.pop('pythonify') - if query.get('date_from'): - query['datefrom'] = self._make_timestamp(query.pop('date_from')) - if query.get('date_to'): - query['dateuntil'] = self._make_timestamp(query.pop('date_to')) - - if query.get('timestamp') is not None: - timestamp = query.pop('timestamp') - if isinstance(timestamp, (list, tuple)): - query['timestamp'] = (self._make_timestamp(timestamp[0]), self._make_timestamp(timestamp[1])) - else: - query['timestamp'] = self._make_timestamp(timestamp) - - url = urljoin(self.root_url, 'events/index') - response = self._prepare_request('POST', url, data=query) - normalized_response = self._check_response(response, expect_json=True) - - if not (self.global_pythonify or pythonify): - return normalized_response - to_return = [] - for e_meta in normalized_response: - me = MISPEvent() - me.from_dict(**e_meta) - to_return.append(me) - return to_return - - def search_sightings(self, context: Optional[str]=None, - context_id: Optional[SearchType]=None, - type_sighting: Optional[str]=None, - date_from: Optional[DateTypes]=None, - date_to: Optional[DateTypes]=None, - publish_timestamp: Optional[DateInterval]=None, last: Optional[DateInterval]=None, - org: Optional[SearchType]=None, - source: Optional[str]=None, - include_attribute: Optional[bool]=None, - include_event_meta: Optional[bool]=None, - pythonify: Optional[bool]=False - ): - '''Search sightings - - :param context: The context of the search. Can be either "attribute", "event", or nothing (will then match on events and attributes). - :param context_id: Only relevant if context is either "attribute" or "event". Then it is the relevant ID. - :param type_sighting: Type of sighting - :param date_from: Events with the date set to a date after the one specified. This filter will use the date of the event. - :param date_to: Events with the date set to a date before the one specified. This filter will use the date of the event. - :param publish_timestamp: Restrict the results by the last publish timestamp (newer than). - :param org: Search by the creator organisation by supplying the organisation identifier. - :param source: Source of the sighting - :param include_attribute: Include the attribute. - :param include_event_meta: Include the meta information of the event. - - Deprecated: - - :param last: synonym for publish_timestamp - - :Example: - - >>> misp.search_sightings(publish_timestamp='30d') # search sightings for the last 30 days on the instance - [ ... ] - >>> misp.search_sightings(context='attribute', context_id=6, include_attribute=True) # return list of sighting for attribute 6 along with the attribute itself - [ ... ] - >>> misp.search_sightings(context='event', context_id=17, include_event_meta=True, org=2) # return list of sighting for event 17 filtered with org id 2 - ''' - query = {'returnFormat': 'json'} - if context is not None: - if context not in ['attribute', 'event']: - raise ValueError('context has to be in {}'.format(', '.join(['attribute', 'event']))) - url_path = f'sightings/restSearch/{context}' - else: - url_path = 'sightings/restSearch' - if isinstance(context_id, (MISPEvent, MISPAttribute)): - context_id = self.__get_uuid_or_id_from_abstract_misp(context_id) - query['id'] = context_id - query['type'] = type_sighting - query['from'] = date_from - query['to'] = date_to - query['last'] = publish_timestamp - query['org_id'] = org - query['source'] = source - query['includeAttribute'] = include_attribute - query['includeEvent'] = include_event_meta - - url = urljoin(self.root_url, url_path) - response = self._prepare_request('POST', url, data=query) - normalized_response = self._check_response(response, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in normalized_response: - return normalized_response - - if self.global_pythonify or pythonify: - to_return = [] - for s in normalized_response: - entries = {} - s_data = s['Sighting'] - if include_event_meta: - e = s_data.pop('Event') - me = MISPEvent() - me.from_dict(**e) - entries['event'] = me - if include_attribute: - a = s_data.pop('Attribute') - ma = MISPAttribute() - ma.from_dict(**a) - entries['attribute'] = ma - ms = MISPSighting() - ms.from_dict(**s_data) - entries['sighting'] = ms - to_return.append(entries) - return to_return - return normalized_response - - def search_logs(self, limit: Optional[int]=None, page: Optional[int]=None, - log_id: Optional[int]=None, title: Optional[str]=None, - created: Optional[DateTypes]=None, model: Optional[str]=None, - action: Optional[str]=None, user_id: Optional[int]=None, - change: Optional[str]=None, email: Optional[str]=None, - org: Optional[str]=None, description: Optional[str]=None, - ip: Optional[str]=None, pythonify: Optional[bool]=False): - '''Search in logs - - Note: to run substring queries simply append/prepend/encapsulate the search term with % - - :param limit: Limit the number of results returned, depending on the scope (for example 10 attributes or 10 full events). - :param page: If a limit is set, sets the page to be returned. page 3, limit 100 will return records 201->300). - :param log_id: Log ID - :param title: Log Title - :param created: Creation timestamp - :param model: Model name that generated the log entry - :param action: The thing that was done - :param user_id: ID of the user doing the action - :param change: Change that occured - :param email: Email of the user - :param org: Organisation of the User doing the action - :param description: Description of the action - :param ip: Origination IP of the User doing the action - :param pythonify: Returns a list of PyMISP Objects instead or the plain json output. Warning: it might use a lot of RAM - ''' - query = locals() - query.pop('self') - query.pop('pythonify') - if log_id is not None: - query['id'] = query.pop('log_id') - - response = self._prepare_request('POST', 'admin/logs/index', data=query) - normalized_response = self._check_response(response, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in normalized_response: - return normalized_response - - to_return = [] - for l in normalized_response: - ml = MISPLog() - ml.from_dict(**l) - to_return.append(ml) - return to_return - - def search_feeds(self, value: Optional[SearchParameterTypes]=None, pythonify: Optional[bool]=False): - '''Search in the feeds cached on the servers''' - response = self._prepare_request('POST', '/feeds/searchCaches', data={'value': value}) - normalized_response = self._check_response(response, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in normalized_response: - return normalized_response - to_return = [] - for feed in normalized_response: - f = MISPFeed() - f.from_dict(**feed) - to_return.append(f) - return to_return - - # ## END Search methods ### - - # ## BEGIN Communities ### - - def communities(self, pythonify: bool=False): - """Get all the communities.""" - communities = self._prepare_request('GET', 'communities') - communities = self._check_response(communities, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in communities: - return communities - to_return = [] - for community in communities: - c = MISPCommunity() - c.from_dict(**community) - to_return.append(c) - return to_return - - def get_community(self, community: Union[MISPCommunity, int, str, UUID], pythonify: bool=False): - '''Get an community from a MISP instance''' - community_id = self.__get_uuid_or_id_from_abstract_misp(community) - community = self._prepare_request('GET', f'communities/view/{community_id}') - community = self._check_response(community, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in community: - return community - c = MISPCommunity() - c.from_dict(**community) - return c - - def request_community_access(self, community: Union[MISPCommunity, int, str, UUID], - requestor_email_address: str=None, - requestor_gpg_key: str=None, - requestor_organisation_name: str=None, - requestor_organisation_uuid: str=None, - requestor_organisation_description: str=None, - message: str=None, sync: bool=False, - anonymise_requestor_server: bool=False, - mock: bool=False): - community_id = self.__get_uuid_or_id_from_abstract_misp(community) - to_post = {'org_name': requestor_organisation_name, - 'org_uuid': requestor_organisation_uuid, - 'org_description': requestor_organisation_description, - 'email': requestor_email_address, 'gpgkey': requestor_gpg_key, - 'message': message, 'anonymise': anonymise_requestor_server, 'sync': sync, - 'mock': mock} - r = self._prepare_request('POST', f'communities/requestAccess/{community_id}', data=to_post) - return self._check_response(r, expect_json=True) - - # ## END Communities ### - - # ## BEGIN Event Delegation ### - - def event_delegations(self, pythonify: bool=False): - """Get all the event delegations.""" - delegations = self._prepare_request('GET', 'event_delegations') - delegations = self._check_response(delegations, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in delegations: - return delegations - to_return = [] - for delegation in delegations: - d = MISPEventDelegation() - d.from_dict(**delegation) - to_return.append(d) - return to_return - - def accept_event_delegation(self, delegation: Union[MISPEventDelegation, int, str], pythonify: bool=False): - delegation_id = self.__get_uuid_or_id_from_abstract_misp(delegation) - delegation = self._prepare_request('POST', f'event_delegations/acceptDelegation/{delegation_id}') - return self._check_response(delegation, expect_json=True) - - def discard_event_delegation(self, delegation: Union[MISPEventDelegation, int, str], pythonify: bool=False): - delegation_id = self.__get_uuid_or_id_from_abstract_misp(delegation) - delegation = self._prepare_request('POST', f'event_delegations/deleteDelegation/{delegation_id}') - return self._check_response(delegation, expect_json=True) - - def delegate_event(self, event: Union[MISPEvent, int, str, UUID]=None, - organisation: Union[MISPOrganisation, int, str, UUID]=None, - event_delegation: MISPEventDelegation=None, - distribution: int=-1, message: str='', pythonify: bool=False): - '''Note: distribution == -1 means recipient decides''' - if event and organisation: - event_id = self.__get_uuid_or_id_from_abstract_misp(event) - organisation_id = self.__get_uuid_or_id_from_abstract_misp(organisation) - if self._old_misp((2, 4, 114), '2020-01-01', sys._getframe().f_code.co_name): - # https://github.com/MISP/MISP/issues/5055 - organisation_id = organisation.id - data = {'event_id': event_id, 'org_id': organisation_id, 'distribution': distribution, 'message': message} - elif event_delegation: - data = event_delegation - else: - raise PyMISPError('Either event and organisation OR event_delegation are required.') - delegation = self._prepare_request('POST', f'event_delegations/delegateEvent/{event_id}', data=data) - delegation = self._check_response(delegation, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in delegation: - return delegation - d = MISPEventDelegation() - d.from_dict(**delegation) - return d - - # ## END Event Delegation ### - - # ## BEGIN Others ### - - def push_event_to_ZMQ(self, event: Union[MISPEvent, int, str, UUID]): - """Force push an event on ZMQ""" - event_id = self.__get_uuid_or_id_from_abstract_misp(event) - response = self._prepare_request('POST', f'events/pushEventToZMQ/{event_id}.json') - return self._check_response(response, expect_json=True) - - def direct_call(self, url: str, data: dict=None, params: dict={}, kw_params: dict={}): - '''Very lightweight call that posts a data blob (python dictionary or json string) on the URL''' - if data is None: - response = self._prepare_request('GET', url, params=params, kw_params=kw_params) - else: - response = self._prepare_request('POST', url, data=data, params=params, kw_params=kw_params) - return self._check_response(response, lenient_response_type=True) - - def freetext(self, event: Union[MISPEvent, int, str, UUID], string: str, adhereToWarninglists: Union[bool, str]=False, - distribution: int=None, returnMetaAttributes: bool=False, pythonify: bool=False, **kwargs): - """Pass a text to the freetext importer""" - event_id = self.__get_uuid_or_id_from_abstract_misp(event) - query = {"value": string} - wl_params = [False, True, 'soft'] - if adhereToWarninglists in wl_params: - query['adhereToWarninglists'] = adhereToWarninglists - else: - raise PyMISPError('Invalid parameter, adhereToWarninglists Can only be {}'.format(', '.join(wl_params))) - if distribution is not None: - query['distribution'] = distribution - if returnMetaAttributes: - query['returnMetaAttributes'] = returnMetaAttributes - attributes = self._prepare_request('POST', f'events/freeTextImport/{event_id}', data=query, **kwargs) - attributes = self._check_response(attributes, expect_json=True) - if returnMetaAttributes or not (self.global_pythonify or pythonify) or 'errors' in attributes: - return attributes - to_return = [] - for attribute in attributes: - a = MISPAttribute() - a.from_dict(**attribute) - to_return.append(a) - return to_return - - def upload_stix(self, path, version: str='2'): - """Upload a STIX file to MISP. - :param path: Path to the STIX on the disk (can be a path-like object, or a pseudofile) - :param version: Can be 1 or 2 - """ - if isinstance(path, (str, Path)): - with open(path, 'rb') as f: - to_post = f.read() - else: - to_post = path.read() - - if isinstance(to_post, bytes): - to_post = to_post.decode() - - if str(version) == '1': - url = urljoin(self.root_url, '/events/upload_stix') - response = self._prepare_request('POST', url, data=to_post, output_type='xml') - else: - url = urljoin(self.root_url, '/events/upload_stix/2') - response = self._prepare_request('POST', url, data=to_post) - - return response - - # ## END Others ### - - # ## BEGIN Statistics ### - - def attributes_statistics(self, context: str='type', percentage: bool=False): - """Get attributes statistics from the MISP instance.""" - # FIXME: https://github.com/MISP/MISP/issues/4874 - if context not in ['type', 'category']: - raise PyMISPError('context can only be "type" or "category"') - if percentage: - path = f'attributes/attributeStatistics/{context}/true' - else: - path = f'attributes/attributeStatistics/{context}' - response = self._prepare_request('GET', path) - return self._check_response(response, expect_json=True) - - def tags_statistics(self, percentage: bool=False, name_sort: bool=False): - """Get tags statistics from the MISP instance""" - # FIXME: https://github.com/MISP/MISP/issues/4874 - # NOTE: https://github.com/MISP/MISP/issues/4879 - if percentage: - percentage = 'true' - else: - percentage = 'false' - if name_sort: - name_sort = 'true' - else: - name_sort = 'false' - response = self._prepare_request('GET', f'tags/tagStatistics/{percentage}/{name_sort}') - return self._check_response(response) - - def users_statistics(self, context: str='data'): - """Get users statistics from the MISP instance""" - availables_contexts = ['data', 'orgs', 'users', 'tags', 'attributehistogram', 'sightings', 'galaxyMatrix'] - if context not in availables_contexts: - raise PyMISPError("context can only be {','.join(availables_contexts)}") - response = self._prepare_request('GET', f'users/statistics/{context}') - return self._check_response(response) - - # ## END Statistics ### - - # ## BEGIN User Settings ### - - def user_settings(self, pythonify: bool=False): - """Get all the user settings.""" - user_settings = self._prepare_request('GET', 'user_settings') - user_settings = self._check_response(user_settings, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in user_settings: - return user_settings - to_return = [] - for user_setting in user_settings: - u = MISPUserSetting() - u.from_dict(**user_setting) - to_return.append(u) - return to_return - - def get_user_setting(self, user_setting: str, user: Union[MISPUser, int, str, UUID]=None, pythonify: bool=False): - '''Get an user setting''' - query = {'setting': user_setting} - if user: - query['user_id'] = self.__get_uuid_or_id_from_abstract_misp(user) - response = self._prepare_request('POST', f'user_settings/getSetting') - user_setting = self._check_response(response, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in user_setting: - return user_setting - u = MISPUserSetting() - u.from_dict(**user_setting) - return u - - def set_user_setting(self, user_setting: str, value: Union[str, dict], user: Union[MISPUser, int, str, UUID]=None, pythonify: bool=False): - '''Get an user setting''' - query = {'setting': user_setting} - if isinstance(value, dict): - value = json.dumps(value) - query['value'] = value - if user: - query['user_id'] = self.__get_uuid_or_id_from_abstract_misp(user) - response = self._prepare_request('POST', f'user_settings/setSetting', data=query) - user_setting = self._check_response(response, expect_json=True) - if not (self.global_pythonify or pythonify) or 'errors' in user_setting: - return user_setting - u = MISPUserSetting() - u.from_dict(**user_setting) - return u - - def delete_user_setting(self, user_setting: str, user: Union[MISPUser, int, str, UUID]=None): - '''Delete a user setting''' - query = {'setting': user_setting} - if user: - query['user_id'] = self.__get_uuid_or_id_from_abstract_misp(user) - response = self._prepare_request('POST', f'user_settings/delete', data=query) - return self._check_response(response, expect_json=True) - - # ## END User Settings ### - - # ## BEGIN Global helpers ### - - def change_sharing_group_on_entity(self, misp_entity: AbstractMISP, sharing_group_id, pythonify: bool=False): - """Change the sharing group of an event, an attribute, or an object""" - misp_entity.distribution = 4 # Needs to be 'Sharing group' - if 'SharingGroup' in misp_entity: # Delete former SharingGroup information - del misp_entity.SharingGroup - misp_entity.sharing_group_id = sharing_group_id # Set new sharing group id - if isinstance(misp_entity, MISPEvent): - return self.update_event(misp_entity, pythonify=pythonify) - - if isinstance(misp_entity, MISPObject): - return self.update_object(misp_entity, pythonify=pythonify) - - if isinstance(misp_entity, MISPAttribute): - return self.update_attribute(misp_entity, pythonify=pythonify) - - raise PyMISPError('The misp_entity must be MISPEvent, MISPObject or MISPAttribute') - - def tag(self, misp_entity: Union[AbstractMISP, str], tag: Union[MISPTag, str], local: bool=False): - """Tag an event or an attribute. misp_entity can be a MISPEvent, a MISP Attribute, or a UUID""" - if 'uuid' in misp_entity: - uuid = misp_entity.uuid - else: - uuid = misp_entity - if isinstance(tag, MISPTag): - tag = tag.name - to_post = {'uuid': uuid, 'tag': tag, 'local': local} - response = self._prepare_request('POST', 'tags/attachTagToObject', data=to_post) - return self._check_response(response, expect_json=True) - - def untag(self, misp_entity: Union[AbstractMISP, str], tag: Union[MISPTag, str]): - """Untag an event or an attribute. misp_entity can be a UUID""" - if 'uuid' in misp_entity: - uuid = misp_entity.uuid - else: - uuid = misp_entity - if isinstance(tag, MISPTag): - tag = tag.name - to_post = {'uuid': uuid, 'tag': tag} - response = self._prepare_request('POST', 'tags/removeTagFromObject', data=to_post) - return self._check_response(response, expect_json=True) - - def build_complex_query(self, or_parameters: Optional[List[SearchType]]=None, - and_parameters: Optional[List[SearchType]]=None, - not_parameters: Optional[List[SearchType]]=None): - '''Build a complex search query. MISP expects a dictionary with AND, OR and NOT keys.''' - to_return = {} - if and_parameters: - to_return['AND'] = and_parameters - if not_parameters: - to_return['NOT'] = not_parameters - if or_parameters: - to_return['OR'] = or_parameters - return to_return - - # ## END Global helpers ### - - # ## Internal methods ### - - def _old_misp(self, minimal_version_required: tuple, removal_date: Union[str, date, datetime], method: str=None, message: str=None): - if self._misp_version >= minimal_version_required: - return False - if isinstance(removal_date, (datetime, date)): - removal_date = removal_date.isoformat() - to_print = f'The instance of MISP you are using is outdated. Unless you update your MISP instance, {method} will stop working after {removal_date}.' - if message: - to_print += f' {message}' - warnings.warn(to_print, DeprecationWarning) - return True - - def __get_uuid_or_id_from_abstract_misp(self, obj: Union[AbstractMISP, int, str, UUID]): - if isinstance(obj, UUID): - return str(obj) - if isinstance(obj, (int, str)): - return obj - - if isinstance(obj, dict) and len(obj.keys()) == 1: - # We have an object in that format: {'Event': {'id': 2, ...}} - # We need to get the content of that dictionary - obj = obj[list(obj.keys())[0]] - - if self._old_misp((2, 4, 113), '2020-01-01', sys._getframe().f_code.co_name, message='MISP now accepts UUIDs to access entiries, usinf it is a lot safer across instances. Just update your MISP instance, plz.'): - if 'id' in obj: - return obj['id'] - if isinstance(obj, MISPShadowAttribute): - # A ShadowAttribute has the same UUID as the related Attribute, we *need* to use the ID - return obj['id'] - if isinstance(obj, MISPEventDelegation): - # An EventDelegation doesn't have a uuid, we *need* to use the ID - return obj['id'] - if 'uuid' in obj: - return obj['uuid'] - return obj['id'] - - def _make_misp_bool(self, parameter: Union[bool, str, None]): - '''MISP wants 0 or 1 for bool, so we avoid True/False '0', '1' ''' - if parameter is None: - return 0 - return 1 if int(parameter) else 0 - - def _make_timestamp(self, value: DateTypes): - '''Catch-all method to normalize anything that can be converted to a timestamp''' - if isinstance(value, datetime): - return value.timestamp() - - if isinstance(value, date): - return datetime.combine(value, datetime.max.time()).timestamp() - - if isinstance(value, str): - if value.isdigit(): - return value - try: - float(value) - return value - except ValueError: - # The value can also be '1d', '10h', ... - return value - return value - - def _check_response(self, response, lenient_response_type=False, expect_json=False): - """Check if the response from the server is not an unexpected error""" - if response.status_code >= 500: - logger.critical(everything_broken.format(response.request.headers, response.request.body, response.text)) - raise MISPServerError(f'Error code 500:\n{response.text}') - - if 400 <= response.status_code < 500: - # The server returns a json message with the error details - try: - error_message = response.json() - except Exception: - raise MISPServerError(f'Error code {response.status_code}:\n{response.text}') - - logger.error(f'Something went wrong ({response.status_code}): {error_message}') - return {'errors': (response.status_code, error_message)} - - # At this point, we had no error. - - try: - response = response.json() - if logger.isEnabledFor(logging.DEBUG): - logger.debug(response) - if isinstance(response, dict) and response.get('response') is not None: - # Cleanup. - response = response['response'] - return response - except Exception: - if logger.isEnabledFor(logging.DEBUG): - logger.debug(response.text) - if expect_json: - raise PyMISPUnexpectedResponse(f'Unexpected response from server: {response.text}') - if lenient_response_type and not response.headers.get('content-type').startswith('application/json'): - return response.text - if not response.content: - # Empty response - logger.error('Got an empty response.') - return {'errors': 'The response is empty.'} - return response.text - - def __repr__(self): - return f'<{self.__class__.__name__}(url={self.root_url})' - - def _prepare_request(self, request_type: str, url: str, data: dict={}, params: dict={}, - kw_params: dict={}, output_type: str='json'): - '''Prepare a request for python-requests''' - url = urljoin(self.root_url, url) - if data: - if not isinstance(data, str): # Else, we already have a text blob to send - if isinstance(data, dict): # Else, we can directly json encode. - # Remove None values. - data = {k: v for k, v in data.items() if v is not None} - data = json.dumps(data, default=pymisp_json_default) - - if logger.isEnabledFor(logging.DEBUG): - logger.debug(f'{request_type} - {url}') - if data is not None: - logger.debug(data) - - if kw_params: - # CakePHP params in URL - to_append_url = '/'.join([f'{k}:{v}' for k, v in kw_params.items()]) - url = f'{url}/{to_append_url}' - req = requests.Request(request_type, url, data=data, params=params) - with requests.Session() as s: - user_agent = f'PyMISP {__version__} - Python {".".join(str(x) for x in sys.version_info[:2])}' - if self.tool: - user_agent = f'{user_agent} - {self.tool}' - req.auth = self.auth - prepped = s.prepare_request(req) - prepped.headers.update( - {'Authorization': self.key, - 'Accept': f'application/{output_type}', - 'content-type': f'application/{output_type}', - 'User-Agent': user_agent}) - if logger.isEnabledFor(logging.DEBUG): - logger.debug(prepped.headers) - settings = s.merge_environment_settings(req.url, proxies=self.proxies or {}, stream=None, verify=self.ssl, cert=self.cert) - return s.send(prepped, **settings) - - def _csv_to_dict(self, csv_content: str): - '''Makes a list of dict out of a csv file (requires headers)''' - fieldnames, lines = csv_content.split('\n', 1) - fieldnames = fieldnames.split(',') - to_return = [] - for line in csv.reader(lines.split('\n')): - if line: - to_return.append({fname: value for fname, value in zip(fieldnames, line)}) - return to_return diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 97b6c0e..10bdb90 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -6,44 +6,18 @@ import os import base64 from io import BytesIO from zipfile import ZipFile -import sys import uuid from collections import defaultdict import logging import hashlib +from pathlib import Path +from typing import List, Optional, Union, IO -from deprecated import deprecated - -from .abstract import AbstractMISP +from .abstract import AbstractMISP, MISPTag from .exceptions import UnknownMISPObjectTemplate, InvalidMISPObject, PyMISPError, NewEventError, NewAttributeError - logger = logging.getLogger('pymisp') -if sys.version_info < (3, 0): - # This is required because Python 2 is a pain. - from datetime import tzinfo, timedelta - - class UTC(tzinfo): - """UTC""" - - def utcoffset(self, dt): - return timedelta(0) - - def tzname(self, dt): - return "UTC" - - def dst(self, dt): - return timedelta(0) - - -if (3, 0) <= sys.version_info < (3, 6): - OLD_PY3 = True -else: - OLD_PY3 = False - -if sys.version_info >= (3, 6): - from pathlib import Path try: from dateutil.parser import parse @@ -69,14 +43,6 @@ except ImportError: except ImportError: has_pyme = False -# Least dirty way to support python 2 and 3 -try: - basestring - unicode -except NameError: - basestring = str - unicode = str - def _int_to_str(d): # transform all integer back to string @@ -102,16 +68,65 @@ def make_bool(value): raise PyMISPError('Unable to convert {} to a boolean.'.format(value)) +class MISPOrganisation(AbstractMISP): + + _fields_for_feed = {'name', 'uuid'} + + def from_dict(self, **kwargs): + if 'Organisation' in kwargs: + kwargs = kwargs['Organisation'] + super(MISPOrganisation, self).from_dict(**kwargs) + + +class MISPShadowAttribute(AbstractMISP): + + def from_dict(self, **kwargs): + if 'ShadowAttribute' in kwargs: + kwargs = kwargs['ShadowAttribute'] + super().from_dict(**kwargs) + + def __repr__(self) -> str: + if hasattr(self, 'value'): + return '<{self.__class__.__name__}(type={self.type}, value={self.value})'.format(self=self) + return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) + + +class MISPSighting(AbstractMISP): + + def from_dict(self, **kwargs): + """Initialize the MISPSighting from a dictionary + :value: Value of the attribute the sighting is related too. Pushing this object + will update the sighting count of each attriutes with thifs value on the instance + :uuid: UUID of the attribute to update + :id: ID of the attriute to update + :source: Source of the sighting + :type: Type of the sighting + :timestamp: Timestamp associated to the sighting + """ + if 'Sighting' in kwargs: + kwargs = kwargs['Sighting'] + super(MISPSighting, self).from_dict(**kwargs) + + def __repr__(self) -> str: + if hasattr(self, 'value'): + return '<{self.__class__.__name__}(value={self.value})'.format(self=self) + if hasattr(self, 'id'): + return '<{self.__class__.__name__}(id={self.id})'.format(self=self) + if hasattr(self, 'uuid'): + return '<{self.__class__.__name__}(uuid={self.uuid})'.format(self=self) + return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) + + class MISPAttribute(AbstractMISP): _fields_for_feed = {'uuid', 'value', 'category', 'type', 'comment', 'data', 'timestamp', 'to_ids', 'disable_correlation'} - def __init__(self, describe_types=None, strict=False): + def __init__(self, describe_types: Optional[dict]=None, strict: bool=False): """Represents an Attribute :describe_type: Use it is you want to overwrite the defualt describeTypes.json file (you don't) :strict: If false, fallback to sane defaults for the attribute type if the ones passed by the user are incorrect """ - super(MISPAttribute, self).__init__() + super().__init__() if describe_types: self.describe_types = describe_types self.__categories = self.describe_types['categories'] @@ -123,7 +138,7 @@ class MISPAttribute(AbstractMISP): self.ShadowAttribute = [] self.Sighting = [] - def hash_values(self, algorithm='sha512'): + def hash_values(self, algorithm: str='sha512') -> List[str]: """Compute the hash of every values for fast lookups""" if algorithm not in hashlib.algorithms_available: raise PyMISPError('The algorithm {} is not available for hashing.'.format(algorithm)) @@ -148,8 +163,8 @@ class MISPAttribute(AbstractMISP): if not hasattr(self, 'timestamp'): self.timestamp = datetime.datetime.timestamp(datetime.datetime.now()) - def _to_feed(self): - to_return = super(MISPAttribute, self)._to_feed() + def _to_feed(self) -> dict: + to_return = super()._to_feed() if self.data: to_return['data'] = base64.b64encode(self.data.getvalue()).decode() if self.tags: @@ -157,23 +172,23 @@ class MISPAttribute(AbstractMISP): return to_return @property - def known_types(self): + def known_types(self) -> List[str]: """Returns a list of all the known MISP attributes types""" return self.describe_types['types'] @property - def malware_binary(self): + def malware_binary(self) -> BytesIO: """Returns a BytesIO of the malware (if the attribute has one, obvs).""" if hasattr(self, '_malware_binary'): return self._malware_binary return None @property - def shadow_attributes(self): + def shadow_attributes(self) -> List[MISPShadowAttribute]: return self.ShadowAttribute @shadow_attributes.setter - def shadow_attributes(self, shadow_attributes): + def shadow_attributes(self, shadow_attributes: List[MISPShadowAttribute]): """Set a list of prepared MISPShadowAttribute.""" if all(isinstance(x, MISPShadowAttribute) for x in shadow_attributes): self.ShadowAttribute = shadow_attributes @@ -181,11 +196,11 @@ class MISPAttribute(AbstractMISP): raise PyMISPError('All the attributes have to be of type MISPShadowAttribute.') @property - def sightings(self): + def sightings(self) -> List[MISPSighting]: return self.Sighting @sightings.setter - def sightings(self, sightings): + def sightings(self, sightings: List[MISPSighting]): """Set a list of prepared MISPShadowAttribute.""" if all(isinstance(x, MISPSighting) for x in sightings): self.Sighting = sightings @@ -196,11 +211,11 @@ class MISPAttribute(AbstractMISP): """Mark the attribute as deleted (soft delete)""" self.deleted = True - def add_proposal(self, shadow_attribute=None, **kwargs): + def add_proposal(self, shadow_attribute=None, **kwargs) -> MISPShadowAttribute: """Alias for add_shadow_attribute""" return self.add_shadow_attribute(shadow_attribute, **kwargs) - def add_shadow_attribute(self, shadow_attribute=None, **kwargs): + def add_shadow_attribute(self, shadow_attribute: Union[MISPShadowAttribute, dict, None]=None, **kwargs) -> MISPShadowAttribute: """Add a shadow attribute to the attribute (by name or a MISPShadowAttribute object)""" if isinstance(shadow_attribute, MISPShadowAttribute): misp_shadow_attribute = shadow_attribute @@ -216,7 +231,7 @@ class MISPAttribute(AbstractMISP): self.edited = True return misp_shadow_attribute - def add_sighting(self, sighting=None, **kwargs): + def add_sighting(self, sighting: Union[MISPSighting, dict, None]=None, **kwargs) -> MISPSighting: """Add a sighting to the attribute (by name or a MISPSighting object)""" if isinstance(sighting, MISPSighting): misp_sighting = sighting @@ -298,10 +313,8 @@ class MISPAttribute(AbstractMISP): ts = kwargs.pop('timestamp') if isinstance(ts, datetime.datetime): self.timestamp = ts - elif sys.version_info >= (3, 3): - self.timestamp = datetime.datetime.fromtimestamp(int(ts), datetime.timezone.utc) else: - self.timestamp = datetime.datetime.fromtimestamp(int(ts), UTC()) + self.timestamp = datetime.datetime.fromtimestamp(int(ts), datetime.timezone.utc) if kwargs.get('sharing_group_id'): self.sharing_group_id = int(kwargs.pop('sharing_group_id')) @@ -325,10 +338,10 @@ class MISPAttribute(AbstractMISP): if self.disable_correlation is None: self.disable_correlation = False - super(MISPAttribute, self).from_dict(**kwargs) + super().from_dict(**kwargs) - def to_dict(self): - to_return = super(MISPAttribute, self).to_dict() + def to_dict(self) -> dict: + to_return = super().to_dict() if self.data: to_return['data'] = base64.b64encode(self.data.getvalue()).decode() return to_return @@ -348,7 +361,7 @@ class MISPAttribute(AbstractMISP): self._malware_binary = self.data self.encrypt = True - def __is_misp_encrypted_file(self, f): + def __is_misp_encrypted_file(self, f) -> bool: files_list = f.namelist() if len(files_list) != 2: return False @@ -368,14 +381,10 @@ class MISPAttribute(AbstractMISP): return self._data if self._data else None @data.setter - def data(self, data): - if sys.version_info <= (3, ): - if isinstance(data, unicode): - self._data = BytesIO(base64.b64decode(data.encode())) - if sys.version_info >= (3, 6): - if isinstance(data, Path): - with data.open('rb') as f: - self._data = BytesIO(f.read()) + def data(self, data: Union[Path, str, bytes, BytesIO]): + if isinstance(data, Path): + with data.open('rb') as f: + self._data = BytesIO(f.read()) if isinstance(data, (str, bytes)): self._data = BytesIO(base64.b64decode(data)) elif isinstance(data, BytesIO): @@ -433,25 +442,305 @@ class MISPAttribute(AbstractMISP): signed, _ = c.sign(to_sign, mode=mode.DETACH) self.sig = base64.b64encode(signed).decode() - @deprecated(reason="Use self.known_types instead. Removal date: 2020-01-01.") - def get_known_types(self): # pragma: no cover - return self.known_types - @deprecated(reason="Use self.malware_binary instead. Removal date: 2020-01-01.") - def get_malware_binary(self): # pragma: no cover - return self.malware_binary +class MISPObjectReference(AbstractMISP): - @deprecated(reason="Use self.to_dict() instead. Removal date: 2020-01-01.") - def _json(self): # pragma: no cover - return self.to_dict() + _fields_for_feed = {'uuid', 'timestamp', 'relationship_type', 'comment', + 'object_uuid', 'referenced_uuid'} - @deprecated(reason="Use self.to_dict() instead. Removal date: 2020-01-01.") - def _json_full(self): # pragma: no cover - return self.to_dict() + def __init__(self): + super().__init__() + self.uuid = str(uuid.uuid4()) - @deprecated(reason="Use self.from_dict(**kwargs) instead. Removal date: 2020-01-01.") - def set_all_values(self, **kwargs): # pragma: no cover - self.from_dict(**kwargs) + def _set_default(self): + if not hasattr(self, 'comment'): + self.comment = '' + if not hasattr(self, 'timestamp'): + self.timestamp = datetime.datetime.timestamp(datetime.datetime.now()) + + def from_dict(self, **kwargs): + if 'ObjectReference' in kwargs: + kwargs = kwargs['ObjectReference'] + super(MISPObjectReference, self).from_dict(**kwargs) + + def __repr__(self) -> str: + if hasattr(self, 'referenced_uuid') and hasattr(self, 'object_uuid'): + return '<{self.__class__.__name__}(object_uuid={self.object_uuid}, referenced_uuid={self.referenced_uuid}, relationship_type={self.relationship_type})'.format(self=self) + return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) + + +class MISPObject(AbstractMISP): + + _fields_for_feed = {'name', 'meta-category', 'description', 'template_uuid', + 'template_version', 'uuid', 'timestamp', 'distribution', + 'sharing_group_id', 'comment'} + + def __init__(self, name: str, strict: bool=False, standalone: bool=False, default_attributes_parameters: dict={}, **kwargs): + ''' Master class representing a generic MISP object + :name: Name of the object + + :strict: Enforce validation with the object templates + + :standalone: The object will be pushed as directly on MISP, not as a part of an event. + In this case the ObjectReference needs to be pushed manually and cannot be in the JSON dump. + + :default_attributes_parameters: Used as template for the attributes if they are not overwritten in add_attribute + + :misp_objects_path_custom: Path to custom object templates + ''' + super().__init__(**kwargs) + self._strict = strict + self.name = name + self._known_template = False + + self._set_template(kwargs.get('misp_objects_path_custom')) + + self.uuid = str(uuid.uuid4()) + self.__fast_attribute_access = defaultdict(list) # Hashtable object_relation: [attributes] + self.ObjectReference = [] + self.Attribute = [] + if isinstance(default_attributes_parameters, MISPAttribute): + # Just make sure we're not modifying an existing MISPAttribute + self._default_attributes_parameters = default_attributes_parameters.to_dict() + else: + self._default_attributes_parameters = default_attributes_parameters + if self._default_attributes_parameters: + # Let's clean that up + self._default_attributes_parameters.pop('value', None) # duh + self._default_attributes_parameters.pop('uuid', None) # duh + self._default_attributes_parameters.pop('id', None) # duh + self._default_attributes_parameters.pop('object_id', None) # duh + self._default_attributes_parameters.pop('type', None) # depends on the value + self._default_attributes_parameters.pop('object_relation', None) # depends on the value + self._default_attributes_parameters.pop('disable_correlation', None) # depends on the value + self._default_attributes_parameters.pop('to_ids', None) # depends on the value + self._default_attributes_parameters.pop('deleted', None) # doesn't make sense to pre-set it + self._default_attributes_parameters.pop('data', None) # in case the original in a sample or an attachment + + # Those values are set for the current object, if they exist, but not pop'd because they are still useful for the attributes + self.distribution = self._default_attributes_parameters.get('distribution', 5) + self.sharing_group_id = self._default_attributes_parameters.get('sharing_group_id', 0) + else: + self.distribution = 5 # Default to inherit + self.sharing_group_id = 0 + self._standalone = standalone + if self._standalone: + # Mark as non_jsonable because we need to add the references manually after the object(s) have been created + self.update_not_jsonable('ObjectReference') + + def _load_template_path(self, template_path: Union[Path, str]) -> bool: + self._definition = self._load_json(template_path) + if not self._definition: + return False + setattr(self, 'meta-category', self._definition['meta-category']) + self.template_uuid = self._definition['uuid'] + self.description = self._definition['description'] + self.template_version = self._definition['version'] + return True + + def _set_default(self): + if not hasattr(self, 'comment'): + self.comment = '' + if not hasattr(self, 'timestamp'): + self.timestamp = datetime.datetime.timestamp(datetime.datetime.now()) + + def _to_feed(self) -> dict: + to_return = super(MISPObject, self)._to_feed() + if self.references: + to_return['ObjectReference'] = [reference._to_feed() for reference in self.references] + return to_return + + def force_misp_objects_path_custom(self, misp_objects_path_custom: Union[Path, str], object_name: Optional[str]=None): + if object_name: + self.name = object_name + self._set_template(misp_objects_path_custom) + + def _set_template(self, misp_objects_path_custom: Union[Path, str]=None): + if misp_objects_path_custom: + # If misp_objects_path_custom is given, and an object with the given name exists, use that. + self.misp_objects_path = misp_objects_path_custom + + # Try to get the template + self._known_template = self._load_template_path(self.misp_objects_path / self.name / 'definition.json') + + if not self._known_template and self._strict: + raise UnknownMISPObjectTemplate('{} is unknown in the MISP object directory.'.format(self.name)) + else: + # Then we have no meta-category, template_uuid, description and template_version + pass + + @property + def disable_validation(self): + self._strict = False + + @property + def attributes(self) -> List[MISPAttribute]: + return self.Attribute + + @attributes.setter + def attributes(self, attributes: List[MISPAttribute]): + if all(isinstance(x, MISPObjectAttribute) for x in attributes): + self.Attribute = attributes + self.__fast_attribute_access = defaultdict(list) + else: + raise PyMISPError('All the attributes have to be of type MISPObjectAttribute.') + + @property + def references(self) -> List[MISPObjectReference]: + return self.ObjectReference + + @references.setter + def references(self, references: List[MISPObjectReference]): + if all(isinstance(x, MISPObjectReference) for x in references): + self.ObjectReference = references + else: + raise PyMISPError('All the attributes have to be of type MISPObjectReference.') + + def from_dict(self, **kwargs): + if 'Object' in kwargs: + kwargs = kwargs['Object'] + if self._known_template: + if kwargs.get('template_uuid') and kwargs['template_uuid'] != self.template_uuid: + if self._strict: + raise UnknownMISPObjectTemplate('UUID of the object is different from the one of the template.') + else: + self._known_template = False + if kwargs.get('template_version') and int(kwargs['template_version']) != self.template_version: + if self._strict: + raise UnknownMISPObjectTemplate('Version of the object ({}) is different from the one of the template ({}).'.format(kwargs['template_version'], self.template_version)) + else: + self._known_template = False + + if 'distribution' in kwargs and kwargs['distribution'] is not None: + self.distribution = kwargs.pop('distribution') + self.distribution = int(self.distribution) + if self.distribution not in [0, 1, 2, 3, 4, 5]: + raise NewAttributeError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 4, 5'.format(self.distribution)) + + if kwargs.get('timestamp'): + ts = kwargs.pop('timestamp') + if isinstance(ts, datetime.datetime): + self.timestamp = ts + else: + self.timestamp = datetime.datetime.fromtimestamp(int(ts), datetime.timezone.utc) + if kwargs.get('Attribute'): + [self.add_attribute(**a) for a in kwargs.pop('Attribute')] + if kwargs.get('ObjectReference'): + [self.add_reference(**r) for r in kwargs.pop('ObjectReference')] + + # Not supported yet - https://github.com/MISP/PyMISP/issues/168 + # if kwargs.get('Tag'): + # for tag in kwargs.pop('Tag'): + # self.add_tag(tag) + + super().from_dict(**kwargs) + + def add_reference(self, referenced_uuid: Union[AbstractMISP, str], relationship_type: str, comment: Optional[str]=None, **kwargs) -> MISPObjectReference: + """Add a link (uuid) to an other object""" + if isinstance(referenced_uuid, AbstractMISP): + # Allow to pass an object or an attribute instead of its UUID + referenced_uuid = referenced_uuid.uuid + if kwargs.get('object_uuid'): + # Load existing object + object_uuid = kwargs.pop('object_uuid') + else: + # New reference + object_uuid = self.uuid + reference = MISPObjectReference() + reference.from_dict(object_uuid=object_uuid, referenced_uuid=referenced_uuid, + relationship_type=relationship_type, comment=comment, **kwargs) + self.ObjectReference.append(reference) + self.edited = True + return reference + + def get_attributes_by_relation(self, object_relation: str) -> List[MISPAttribute]: + '''Returns the list of attributes with the given object relation in the object''' + return self._fast_attribute_access.get(object_relation, []) + + @property + def _fast_attribute_access(self): + if not self.__fast_attribute_access: + for a in self.attributes: + self.__fast_attribute_access[a.object_relation].append(a) + return self.__fast_attribute_access + + def has_attributes_by_relation(self, list_of_relations: List[str]): + '''True if all the relations in the list are defined in the object''' + return all(relation in self._fast_attribute_access for relation in list_of_relations) + + def add_attribute(self, object_relation: str, simple_value: Union[str, int, float]=None, **value) -> MISPAttribute: + """Add an attribute. object_relation is required and the value key is a + dictionary with all the keys supported by MISPAttribute""" + if simple_value is not None: # /!\ The value *can* be 0 + value = {'value': simple_value} + if value.get('value') in [None, '']: + logger.warning("The value of the attribute you're trying to add is None or empty string, skipping it. Object relation: {}".format(object_relation)) + return None + if self._known_template: + if object_relation in self._definition['attributes']: + attribute = MISPObjectAttribute(self._definition['attributes'][object_relation]) + else: + # Woopsie, this object_relation is unknown, no sane defaults for you. + logger.warning("The template ({}) doesn't have the object_relation ({}) you're trying to add.".format(self.name, object_relation)) + attribute = MISPObjectAttribute({}) + else: + attribute = MISPObjectAttribute({}) + # Overwrite the parameters of self._default_attributes_parameters with the ones of value + attribute.from_dict(object_relation=object_relation, **dict(self._default_attributes_parameters, **value)) + # FIXME New syntax python3 only, keep for later. + # attribute.from_dict(object_relation=object_relation, **{**self._default_attributes_parameters, **value}) + self.__fast_attribute_access[object_relation].append(attribute) + self.Attribute.append(attribute) + self.edited = True + return attribute + + def add_attributes(self, object_relation: str, *attributes) -> List[MISPAttribute]: + '''Add multiple attributes with the same object_relation. + Helper for object_relation when multiple is True in the template. + It is the same as calling multiple times add_attribute with the same object_relation. + ''' + to_return = [] + for attribute in attributes: + if isinstance(attribute, dict): + a = self.add_attribute(object_relation, **attribute) + else: + a = self.add_attribute(object_relation, value=attribute) + to_return.append(a) + return to_return + + def to_dict(self, strict: bool=False) -> dict: + if strict or self._strict and self._known_template: + self._validate() + return super(MISPObject, self).to_dict() + + def to_json(self, strict: bool=False, sort_keys: bool=False, indent: Optional[int]=None): + if strict or self._strict and self._known_template: + self._validate() + return super(MISPObject, self).to_json(sort_keys=sort_keys, indent=indent) + + def _validate(self): + """Make sure the object we're creating has the required fields""" + if self._definition.get('required'): + required_missing = set(self._definition.get('required')) - set(self._fast_attribute_access.keys()) + if required_missing: + raise InvalidMISPObject('{} are required.'.format(required_missing)) + if self._definition.get('requiredOneOf'): + if not set(self._definition['requiredOneOf']) & set(self._fast_attribute_access.keys()): + # We ecpect at least one of the object_relation in requiredOneOf, and it isn't the case + raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self._definition['requiredOneOf']))) + for rel, attrs in self._fast_attribute_access.items(): + if len(attrs) == 1: + # object_relation's here only once, everything's cool, moving on + continue + if not self._definition['attributes'][rel].get('multiple'): + # object_relation's here more than once, but it isn't allowed in the template. + raise InvalidMISPObject('Multiple occurrences of {} is not allowed'.format(rel)) + return True + + def __repr__(self) -> str: + if hasattr(self, 'name'): + return '<{self.__class__.__name__}(name={self.name})'.format(self=self) + return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) class MISPEvent(AbstractMISP): @@ -459,16 +748,13 @@ class MISPEvent(AbstractMISP): _fields_for_feed = {'uuid', 'info', 'threat_level_id', 'analysis', 'timestamp', 'publish_timestamp', 'published', 'date', 'extends_uuid'} - def __init__(self, describe_types=None, strict_validation=False, **kwargs): - super(MISPEvent, self).__init__(**kwargs) + def __init__(self, describe_types: dict=None, strict_validation: bool=False, **kwargs): + super().__init__(**kwargs) if strict_validation: schema_file = 'schema.json' else: schema_file = 'schema-lax.json' - if sys.version_info >= (3, 4): - self.__json_schema = self._load_json(self.resources_path / schema_file) - else: - self.__json_schema = self._load_json(os.path.join(self.resources_path, schema_file)) + self.__json_schema = self._load_json(self.resources_path / schema_file) if describe_types: # This variable is used in add_attribute in order to avoid duplicating the structure self.describe_types = describe_types @@ -500,7 +786,7 @@ class MISPEvent(AbstractMISP): self.threat_level_id = 4 @property - def manifest(self): + def manifest(self) -> dict: required = ['info', 'Orgc'] for r in required: if not hasattr(self, r): @@ -520,7 +806,7 @@ class MISPEvent(AbstractMISP): } } - def attributes_hashes(self, algorithm='sha512'): + def attributes_hashes(self, algorithm: str='sha512') -> List[str]: to_return = [] for attribute in self.attributes: to_return += attribute.hash_values(algorithm) @@ -529,7 +815,7 @@ class MISPEvent(AbstractMISP): to_return += attribute.hash_values(algorithm) return to_return - def to_feed(self, valid_distributions=[0, 1, 2, 3, 4, 5], with_meta=False): + def to_feed(self, valid_distributions: List[int]=[0, 1, 2, 3, 4, 5], with_meta: bool=False) -> dict: """ Generate a json output for MISP Feed. Notes: * valid_distributions only makes sense if the distribution key is set (i.e. the event is exported from a MISP instance) @@ -544,7 +830,7 @@ class MISPEvent(AbstractMISP): and int(self.distribution) not in valid_distributions): return - to_return = super(MISPEvent, self)._to_feed() + to_return = super()._to_feed() if with_meta: to_return['_hashes'] = [] to_return['_manifest'] = self.manifest @@ -578,76 +864,74 @@ class MISPEvent(AbstractMISP): return {'Event': to_return} @property - def known_types(self): + def known_types(self) -> List[str]: return self.describe_types['types'] @property - def org(self): + def org(self) -> MISPOrganisation: return self.Org @property - def orgc(self): + def orgc(self) -> MISPOrganisation: return self.Orgc @orgc.setter - def orgc(self, orgc): + def orgc(self, orgc: MISPOrganisation): if isinstance(orgc, MISPOrganisation): self.Orgc = orgc else: raise PyMISPError('Orgc must be of type MISPOrganisation.') @property - def attributes(self): + def attributes(self) -> List[MISPAttribute]: return self.Attribute @attributes.setter - def attributes(self, attributes): + def attributes(self, attributes: List[MISPAttribute]): if all(isinstance(x, MISPAttribute) for x in attributes): self.Attribute = attributes else: raise PyMISPError('All the attributes have to be of type MISPAttribute.') @property - def shadow_attributes(self): + def shadow_attributes(self) -> List[MISPShadowAttribute]: return self.ShadowAttribute @shadow_attributes.setter - def shadow_attributes(self, shadow_attributes): + def shadow_attributes(self, shadow_attributes: List[MISPShadowAttribute]): if all(isinstance(x, MISPShadowAttribute) for x in shadow_attributes): self.ShadowAttribute = shadow_attributes else: raise PyMISPError('All the attributes have to be of type MISPShadowAttribute.') @property - def related_events(self): + def related_events(self): # -> List[MISPEvent]: return self.RelatedEvent @property - def objects(self): + def objects(self) -> List[MISPObject]: return self.Object @objects.setter - def objects(self, objects): + def objects(self, objects: List[MISPObject]): if all(isinstance(x, MISPObject) for x in objects): self.Object = objects else: raise PyMISPError('All the attributes have to be of type MISPObject.') - def load_file(self, event_path, validate=False, metadata_only=False): + def load_file(self, event_path: Union[Path, str], validate: bool=False, metadata_only: bool=False): """Load a JSON dump from a file on the disk""" if not os.path.exists(event_path): raise PyMISPError('Invalid path, unable to load the event.') with open(event_path, 'rb') as f: self.load(f, validate, metadata_only) - def load(self, json_event, validate=False, metadata_only=False): + def load(self, json_event: Union[IO, str, bytes], validate: bool=False, metadata_only: bool=False): """Load a JSON dump from a pseudo file or a JSON string""" if hasattr(json_event, 'read'): # python2 and python3 compatible to find if we have a file json_event = json_event.read() - if isinstance(json_event, (basestring, bytes)): - if OLD_PY3 and isinstance(json_event, bytes): - json_event = json_event.decode() + if isinstance(json_event, (str, bytes)): json_event = json.loads(json_event) if json_event.get('response'): event = json_event.get('response')[0] @@ -662,9 +946,9 @@ class MISPEvent(AbstractMISP): if validate: jsonschema.validate(json.loads(self.to_json()), self.__json_schema) - def set_date(self, date, ignore_invalid=False): + def set_date(self, date: Union[str, int, datetime.datetime, datetime.date, None], ignore_invalid: bool=False): """Set a date for the event (string, datetime, or date object)""" - if isinstance(date, basestring) or isinstance(date, unicode): + if isinstance(date, str): self.date = parse(date).date() elif isinstance(date, int): self.date = datetime.datetime.utcfromtimestamp(date).date() @@ -722,20 +1006,11 @@ class MISPEvent(AbstractMISP): if kwargs.get('org_id'): self.org_id = int(kwargs.pop('org_id')) if kwargs.get('timestamp'): - if sys.version_info >= (3, 3): - self.timestamp = datetime.datetime.fromtimestamp(int(kwargs.pop('timestamp')), datetime.timezone.utc) - else: - self.timestamp = datetime.datetime.fromtimestamp(int(kwargs.pop('timestamp')), UTC()) + self.timestamp = datetime.datetime.fromtimestamp(int(kwargs.pop('timestamp')), datetime.timezone.utc) if kwargs.get('publish_timestamp'): - if sys.version_info >= (3, 3): - self.publish_timestamp = datetime.datetime.fromtimestamp(int(kwargs.pop('publish_timestamp')), datetime.timezone.utc) - else: - self.publish_timestamp = datetime.datetime.fromtimestamp(int(kwargs.pop('publish_timestamp')), UTC()) + self.publish_timestamp = datetime.datetime.fromtimestamp(int(kwargs.pop('publish_timestamp')), datetime.timezone.utc) if kwargs.get('sighting_timestamp'): - if sys.version_info >= (3, 3): - self.sighting_timestamp = datetime.datetime.fromtimestamp(int(kwargs.pop('sighting_timestamp')), datetime.timezone.utc) - else: - self.sighting_timestamp = datetime.datetime.fromtimestamp(int(kwargs.pop('sighting_timestamp')), UTC()) + self.sighting_timestamp = datetime.datetime.fromtimestamp(int(kwargs.pop('sighting_timestamp')), datetime.timezone.utc) if kwargs.get('sharing_group_id'): self.sharing_group_id = int(kwargs.pop('sharing_group_id')) if kwargs.get('RelatedEvent'): @@ -756,8 +1031,8 @@ class MISPEvent(AbstractMISP): super(MISPEvent, self).from_dict(**kwargs) - def to_dict(self): - to_return = super(MISPEvent, self).to_dict() + def to_dict(self) -> dict: + to_return = super().to_dict() if to_return.get('date'): if isinstance(self.date, datetime.datetime): @@ -770,11 +1045,11 @@ class MISPEvent(AbstractMISP): return to_return - def add_proposal(self, shadow_attribute=None, **kwargs): + def add_proposal(self, shadow_attribute=None, **kwargs) -> MISPShadowAttribute: """Alias for add_shadow_attribute""" return self.add_shadow_attribute(shadow_attribute, **kwargs) - def add_shadow_attribute(self, shadow_attribute=None, **kwargs): + def add_shadow_attribute(self, shadow_attribute=None, **kwargs) -> MISPShadowAttribute: """Add a tag to the attribute (by name or a MISPTag object)""" if isinstance(shadow_attribute, MISPShadowAttribute): misp_shadow_attribute = shadow_attribute @@ -790,7 +1065,7 @@ class MISPEvent(AbstractMISP): self.edited = True return misp_shadow_attribute - def get_attribute_tag(self, attribute_identifier): + def get_attribute_tag(self, attribute_identifier: str) -> List[MISPTag]: """Return the tags associated to an attribute or an object attribute. :attribute_identifier: can be an ID, UUID, or the value. """ @@ -803,7 +1078,7 @@ class MISPEvent(AbstractMISP): tags += a.tags return tags - def add_attribute_tag(self, tag, attribute_identifier): + def add_attribute_tag(self, tag: Union[MISPTag, str], attribute_identifier: str) -> List[MISPAttribute]: """Add a tag to an existing attribute, raise an Exception if the attribute doesn't exists. :tag: Tag name as a string, MISPTag instance, or dictionary :attribute_identifier: can be an ID, UUID, or the value. @@ -830,7 +1105,7 @@ class MISPEvent(AbstractMISP): """Mark the attribute as un-published (set publish flag to false)""" self.published = False - def delete_attribute(self, attribute_id): + def delete_attribute(self, attribute_id: str): """Delete an attribute, you can search by ID or UUID""" found = False for a in self.attributes: @@ -842,7 +1117,7 @@ class MISPEvent(AbstractMISP): if not found: raise PyMISPError('No attribute with UUID/ID {} found.'.format(attribute_id)) - def add_attribute(self, type, value, **kwargs): + def add_attribute(self, type: str, value: Union[str, int, float], **kwargs) -> MISPAttribute: """Add an attribute. type and value are required but you can pass all other parameters supported by MISPAttribute""" attr_list = [] @@ -857,21 +1132,21 @@ class MISPEvent(AbstractMISP): return attr_list return attribute - def get_object_by_id(self, object_id): + def get_object_by_id(self, object_id: Union[str, int]) -> MISPObject: """Get an object by ID (the ID is the one set by the server when creating the new object)""" for obj in self.objects: if hasattr(obj, 'id') and int(obj.id) == int(object_id): return obj raise InvalidMISPObject('Object with {} does not exist in this event'.format(object_id)) - def get_object_by_uuid(self, object_uuid): + def get_object_by_uuid(self, object_uuid: str) -> MISPObject: """Get an object by UUID (UUID is set by the server when creating the new object)""" for obj in self.objects: if hasattr(obj, 'uuid') and obj.uuid == object_uuid: return obj raise InvalidMISPObject('Object with {} does not exist in this event'.format(object_uuid)) - def get_objects_by_name(self, object_name): + def get_objects_by_name(self, object_name: str) -> List[MISPObject]: """Get an object by UUID (UUID is set by the server when creating the new object)""" objects = [] for obj in self.objects: @@ -879,7 +1154,7 @@ class MISPEvent(AbstractMISP): objects.append(obj) return objects - def add_object(self, obj=None, **kwargs): + def add_object(self, obj: Union[MISPObject, dict, None]=None, **kwargs) -> MISPObject: """Add an object to the Event, either by passing a MISPObject, or a dictionary""" if isinstance(obj, MISPObject): misp_obj = obj @@ -900,8 +1175,6 @@ class MISPEvent(AbstractMISP): return misp_obj def run_expansions(self): - if sys.version_info < (3, 6): - raise PyMISPError("No, seriously, ain't gonna work with python <=3.6") for index, attribute in enumerate(self.attributes): if 'expand' not in attribute: continue @@ -924,7 +1197,7 @@ class MISPEvent(AbstractMISP): else: logger.warning('No expansions for this data type ({}). Open an issue if needed.'.format(attribute.type)) - def __repr__(self): + def __repr__(self) -> str: if hasattr(self, 'info'): return '<{self.__class__.__name__}(info={self.info})'.format(self=self) return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) @@ -989,242 +1262,129 @@ class MISPEvent(AbstractMISP): to_return['global'] = False return to_return - @deprecated(reason="Use self.known_types instead. Removal date: 2020-01-01.") - def get_known_types(self): # pragma: no cover - return self.known_types - - @deprecated(reason="Use self.from_dict(**kwargs) instead. Removal date: 2020-01-01.") - def set_all_values(self, **kwargs): # pragma: no cover - self.from_dict(**kwargs) - - @deprecated(reason="Use self.to_dict() instead. Removal date: 2020-01-01.") - def _json(self): # pragma: no cover - return self.to_dict() - - -class MISPObjectReference(AbstractMISP): - - _fields_for_feed = {'uuid', 'timestamp', 'relationship_type', 'comment', - 'object_uuid', 'referenced_uuid'} - - def __init__(self): - super(MISPObjectReference, self).__init__() - self.uuid = str(uuid.uuid4()) - - def _set_default(self): - if not hasattr(self, 'comment'): - self.comment = '' - if not hasattr(self, 'timestamp'): - self.timestamp = datetime.datetime.timestamp(datetime.datetime.now()) - - def from_dict(self, **kwargs): - if 'ObjectReference' in kwargs: - kwargs = kwargs['ObjectReference'] - super(MISPObjectReference, self).from_dict(**kwargs) - - def __repr__(self): - if hasattr(self, 'referenced_uuid') and hasattr(self, 'object_uuid'): - return '<{self.__class__.__name__}(object_uuid={self.object_uuid}, referenced_uuid={self.referenced_uuid}, relationship_type={self.relationship_type})'.format(self=self) - return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) - class MISPObjectTemplate(AbstractMISP): - def __init__(self): - super(MISPObjectTemplate, self).__init__() - def from_dict(self, **kwargs): if 'ObjectTemplate' in kwargs: kwargs = kwargs['ObjectTemplate'] - super(MISPObjectTemplate, self).from_dict(**kwargs) + super().from_dict(**kwargs) + + def __repr__(self) -> str: + return '<{self.__class__.__name__}(self.name)'.format(self=self) class MISPUser(AbstractMISP): - def __init__(self): - super(MISPUser, self).__init__() - def from_dict(self, **kwargs): if 'User' in kwargs: kwargs = kwargs['User'] - super(MISPUser, self).from_dict(**kwargs) + super().from_dict(**kwargs) if hasattr(self, 'password') and set(self.password) == set(['*']): self.password = None - def __repr__(self): + def __repr__(self) -> str: if hasattr(self, 'email'): return '<{self.__class__.__name__}(email={self.email})'.format(self=self) return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) -class MISPOrganisation(AbstractMISP): - - _fields_for_feed = {'name', 'uuid'} - - def __init__(self): - super(MISPOrganisation, self).__init__() - - def from_dict(self, **kwargs): - if 'Organisation' in kwargs: - kwargs = kwargs['Organisation'] - super(MISPOrganisation, self).from_dict(**kwargs) - - class MISPFeed(AbstractMISP): - def __init__(self): - super(MISPFeed, self).__init__() - def from_dict(self, **kwargs): if 'Feed' in kwargs: kwargs = kwargs['Feed'] - super(MISPFeed, self).from_dict(**kwargs) + super().from_dict(**kwargs) class MISPWarninglist(AbstractMISP): - def __init__(self): - super(MISPWarninglist, self).__init__() - def from_dict(self, **kwargs): if 'Warninglist' in kwargs: kwargs = kwargs['Warninglist'] - super(MISPWarninglist, self).from_dict(**kwargs) + super().from_dict(**kwargs) class MISPTaxonomy(AbstractMISP): - def __init__(self): - super(MISPTaxonomy, self).__init__() - def from_dict(self, **kwargs): if 'Taxonomy' in kwargs: kwargs = kwargs['Taxonomy'] - super(MISPTaxonomy, self).from_dict(**kwargs) + super().from_dict(**kwargs) class MISPGalaxy(AbstractMISP): - def __init__(self): - super(MISPGalaxy, self).__init__() - def from_dict(self, **kwargs): if 'Galaxy' in kwargs: kwargs = kwargs['Galaxy'] - super(MISPGalaxy, self).from_dict(**kwargs) + super().from_dict(**kwargs) class MISPNoticelist(AbstractMISP): - def __init__(self): - super(MISPNoticelist, self).__init__() - def from_dict(self, **kwargs): if 'Noticelist' in kwargs: kwargs = kwargs['Noticelist'] - super(MISPNoticelist, self).from_dict(**kwargs) + super().from_dict(**kwargs) class MISPRole(AbstractMISP): - def __init__(self): - super(MISPRole, self).__init__() - def from_dict(self, **kwargs): if 'Role' in kwargs: kwargs = kwargs['Role'] - super(MISPRole, self).from_dict(**kwargs) + super().from_dict(**kwargs) class MISPServer(AbstractMISP): - def __init__(self): - super(MISPServer, self).__init__() - def from_dict(self, **kwargs): if 'Server' in kwargs: kwargs = kwargs['Server'] - super(MISPServer, self).from_dict(**kwargs) + super().from_dict(**kwargs) class MISPSharingGroup(AbstractMISP): - def __init__(self): - super(MISPSharingGroup, self).__init__() - def from_dict(self, **kwargs): if 'SharingGroup' in kwargs: kwargs = kwargs['SharingGroup'] - super(MISPSharingGroup, self).from_dict(**kwargs) + super().from_dict(**kwargs) class MISPLog(AbstractMISP): - def __init__(self): - super(MISPLog, self).__init__() - def from_dict(self, **kwargs): if 'Log' in kwargs: kwargs = kwargs['Log'] - super(MISPLog, self).from_dict(**kwargs) + super().from_dict(**kwargs) - def __repr__(self): + def __repr__(self) -> str: return '<{self.__class__.__name__}({self.model}, {self.action}, {self.title})'.format(self=self) class MISPEventDelegation(AbstractMISP): - def __init__(self): - super(MISPEventDelegation, self).__init__() - def from_dict(self, **kwargs): if 'EventDelegation' in kwargs: kwargs = kwargs['EventDelegation'] - super(MISPEventDelegation, self).from_dict(**kwargs) + super().from_dict(**kwargs) - def __repr__(self): + def __repr__(self) -> str: return '<{self.__class__.__name__}(org_id={self.org_id}, requester_org_id={self.requester_org_id}, {self.event_id})'.format(self=self) -class MISPSighting(AbstractMISP): - - def __init__(self): - super(MISPSighting, self).__init__() - - def from_dict(self, **kwargs): - """Initialize the MISPSighting from a dictionary - :value: Value of the attribute the sighting is related too. Pushing this object - will update the sighting count of each attriutes with thifs value on the instance - :uuid: UUID of the attribute to update - :id: ID of the attriute to update - :source: Source of the sighting - :type: Type of the sighting - :timestamp: Timestamp associated to the sighting - """ - if 'Sighting' in kwargs: - kwargs = kwargs['Sighting'] - super(MISPSighting, self).from_dict(**kwargs) - - def __repr__(self): - if hasattr(self, 'value'): - return '<{self.__class__.__name__}(value={self.value})'.format(self=self) - if hasattr(self, 'id'): - return '<{self.__class__.__name__}(id={self.id})'.format(self=self) - if hasattr(self, 'uuid'): - return '<{self.__class__.__name__}(uuid={self.uuid})'.format(self=self) - return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) - - class MISPObjectAttribute(MISPAttribute): _fields_for_feed = {'uuid', 'object_relation', 'value', 'category', 'type', 'comment', 'data', 'timestamp', 'to_ids', 'disable_correlation'} def __init__(self, definition): - super(MISPObjectAttribute, self).__init__() + super().__init__() self._definition = definition - def from_dict(self, object_relation, value, **kwargs): + def from_dict(self, object_relation: str, value: Union[str, int, float], **kwargs): self.object_relation = object_relation self.value = value if 'Attribute' in kwargs: @@ -1248,9 +1408,7 @@ class MISPObjectAttribute(MISPAttribute): self.to_ids = self._definition.get('to_ids') if not self.type: raise NewAttributeError("The type of the attribute is required. Is the object template missing?") - super(MISPObjectAttribute, self).from_dict(**dict(self, **kwargs)) - # FIXME New syntax python3 only, keep for later. - # super(MISPObjectAttribute, self).from_dict(**{**self, **kwargs}) + super().from_dict(**{**self, **kwargs}) def __repr__(self): if hasattr(self, 'value'): @@ -1258,31 +1416,12 @@ class MISPObjectAttribute(MISPAttribute): return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) -class MISPShadowAttribute(AbstractMISP): - - def __init__(self): - super(MISPShadowAttribute, self).__init__() - - def from_dict(self, **kwargs): - if 'ShadowAttribute' in kwargs: - kwargs = kwargs['ShadowAttribute'] - super(MISPShadowAttribute, self).from_dict(**kwargs) - - def __repr__(self): - if hasattr(self, 'value'): - return '<{self.__class__.__name__}(type={self.type}, value={self.value})'.format(self=self) - return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) - - class MISPCommunity(AbstractMISP): - def __init__(self): - super(MISPCommunity, self).__init__() - def from_dict(self, **kwargs): if 'Community' in kwargs: kwargs = kwargs['Community'] - super(MISPCommunity, self).from_dict(**kwargs) + super().from_dict(**kwargs) def __repr__(self): return '<{self.__class__.__name__}(name={self.name}, uuid={self.uuid})'.format(self=self) @@ -1290,292 +1429,10 @@ class MISPCommunity(AbstractMISP): class MISPUserSetting(AbstractMISP): - def __init__(self): - super(MISPUserSetting, self).__init__() - def from_dict(self, **kwargs): if 'UserSetting' in kwargs: kwargs = kwargs['UserSetting'] - super(MISPUserSetting, self).from_dict(**kwargs) + super().from_dict(**kwargs) def __repr__(self): return '<{self.__class__.__name__}(name={self.setting}'.format(self=self) - - -class MISPObject(AbstractMISP): - - _fields_for_feed = {'name', 'meta-category', 'description', 'template_uuid', - 'template_version', 'uuid', 'timestamp', 'distribution', - 'sharing_group_id', 'comment'} - - def __init__(self, name, strict=False, standalone=False, default_attributes_parameters={}, **kwargs): - ''' Master class representing a generic MISP object - :name: Name of the object - - :strict: Enforce validation with the object templates - - :standalone: The object will be pushed as directly on MISP, not as a part of an event. - In this case the ObjectReference needs to be pushed manually and cannot be in the JSON dump. - - :default_attributes_parameters: Used as template for the attributes if they are not overwritten in add_attribute - - :misp_objects_path_custom: Path to custom object templates - ''' - super(MISPObject, self).__init__(**kwargs) - self._strict = strict - self.name = name - self._known_template = False - - self._set_template(kwargs.get('misp_objects_path_custom')) - - self.uuid = str(uuid.uuid4()) - self.__fast_attribute_access = defaultdict(list) # Hashtable object_relation: [attributes] - self.ObjectReference = [] - self.Attribute = [] - if isinstance(default_attributes_parameters, MISPAttribute): - # Just make sure we're not modifying an existing MISPAttribute - self._default_attributes_parameters = default_attributes_parameters.to_dict() - else: - self._default_attributes_parameters = default_attributes_parameters - if self._default_attributes_parameters: - # Let's clean that up - self._default_attributes_parameters.pop('value', None) # duh - self._default_attributes_parameters.pop('uuid', None) # duh - self._default_attributes_parameters.pop('id', None) # duh - self._default_attributes_parameters.pop('object_id', None) # duh - self._default_attributes_parameters.pop('type', None) # depends on the value - self._default_attributes_parameters.pop('object_relation', None) # depends on the value - self._default_attributes_parameters.pop('disable_correlation', None) # depends on the value - self._default_attributes_parameters.pop('to_ids', None) # depends on the value - self._default_attributes_parameters.pop('deleted', None) # doesn't make sense to pre-set it - self._default_attributes_parameters.pop('data', None) # in case the original in a sample or an attachment - - # Those values are set for the current object, if they exist, but not pop'd because they are still useful for the attributes - self.distribution = self._default_attributes_parameters.get('distribution', 5) - self.sharing_group_id = self._default_attributes_parameters.get('sharing_group_id', 0) - else: - self.distribution = 5 # Default to inherit - self.sharing_group_id = 0 - self._standalone = standalone - if self._standalone: - # Mark as non_jsonable because we need to add the references manually after the object(s) have been created - self.update_not_jsonable('ObjectReference') - - def _load_template_path(self, template_path): - self._definition = self._load_json(template_path) - if not self._definition: - return False - setattr(self, 'meta-category', self._definition['meta-category']) - self.template_uuid = self._definition['uuid'] - self.description = self._definition['description'] - self.template_version = self._definition['version'] - return True - - def _set_default(self): - if not hasattr(self, 'comment'): - self.comment = '' - if not hasattr(self, 'timestamp'): - self.timestamp = datetime.datetime.timestamp(datetime.datetime.now()) - - def _to_feed(self): - to_return = super(MISPObject, self)._to_feed() - if self.references: - to_return['ObjectReference'] = [reference._to_feed() for reference in self.references] - return to_return - - def force_misp_objects_path_custom(self, misp_objects_path_custom, object_name=None): - if object_name: - self.name = object_name - self._set_template(misp_objects_path_custom) - - def _set_template(self, misp_objects_path_custom=None): - if misp_objects_path_custom: - # If misp_objects_path_custom is given, and an object with the given name exists, use that. - self.misp_objects_path = misp_objects_path_custom - - # Try to get the template - if sys.version_info >= (3, 4): - self._known_template = self._load_template_path(self.misp_objects_path / self.name / 'definition.json') - else: - self._known_template = self._load_template_path(os.path.join(self.misp_objects_path, self.name, 'definition.json')) - - if not self._known_template and self._strict: - raise UnknownMISPObjectTemplate('{} is unknown in the MISP object directory.'.format(self.name)) - else: - # Then we have no meta-category, template_uuid, description and template_version - pass - - @property - def disable_validation(self): - self._strict = False - - @property - def attributes(self): - return self.Attribute - - @attributes.setter - def attributes(self, attributes): - if all(isinstance(x, MISPObjectAttribute) for x in attributes): - self.Attribute = attributes - self.__fast_attribute_access = defaultdict(list) - else: - raise PyMISPError('All the attributes have to be of type MISPObjectAttribute.') - - @property - def references(self): - return self.ObjectReference - - @references.setter - def references(self, references): - if all(isinstance(x, MISPObjectReference) for x in references): - self.ObjectReference = references - else: - raise PyMISPError('All the attributes have to be of type MISPObjectReference.') - - def from_dict(self, **kwargs): - if 'Object' in kwargs: - kwargs = kwargs['Object'] - if self._known_template: - if kwargs.get('template_uuid') and kwargs['template_uuid'] != self.template_uuid: - if self._strict: - raise UnknownMISPObjectTemplate('UUID of the object is different from the one of the template.') - else: - self._known_template = False - if kwargs.get('template_version') and int(kwargs['template_version']) != self.template_version: - if self._strict: - raise UnknownMISPObjectTemplate('Version of the object ({}) is different from the one of the template ({}).'.format(kwargs['template_version'], self.template_version)) - else: - self._known_template = False - - if 'distribution' in kwargs and kwargs['distribution'] is not None: - self.distribution = kwargs.pop('distribution') - self.distribution = int(self.distribution) - if self.distribution not in [0, 1, 2, 3, 4, 5]: - raise NewAttributeError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 4, 5'.format(self.distribution)) - - if kwargs.get('timestamp'): - ts = kwargs.pop('timestamp') - if isinstance(ts, datetime.datetime): - self.timestamp = ts - elif sys.version_info >= (3, 3): - self.timestamp = datetime.datetime.fromtimestamp(int(ts), datetime.timezone.utc) - else: - self.timestamp = datetime.datetime.fromtimestamp(int(ts), UTC()) - if kwargs.get('Attribute'): - [self.add_attribute(**a) for a in kwargs.pop('Attribute')] - if kwargs.get('ObjectReference'): - [self.add_reference(**r) for r in kwargs.pop('ObjectReference')] - - # Not supported yet - https://github.com/MISP/PyMISP/issues/168 - # if kwargs.get('Tag'): - # for tag in kwargs.pop('Tag'): - # self.add_tag(tag) - - super(MISPObject, self).from_dict(**kwargs) - - def add_reference(self, referenced_uuid, relationship_type, comment=None, **kwargs): - """Add a link (uuid) to an other object""" - if isinstance(referenced_uuid, AbstractMISP): - # Allow to pass an object or an attribute instead of its UUID - referenced_uuid = referenced_uuid.uuid - if kwargs.get('object_uuid'): - # Load existing object - object_uuid = kwargs.pop('object_uuid') - else: - # New reference - object_uuid = self.uuid - reference = MISPObjectReference() - reference.from_dict(object_uuid=object_uuid, referenced_uuid=referenced_uuid, - relationship_type=relationship_type, comment=comment, **kwargs) - self.ObjectReference.append(reference) - self.edited = True - return reference - - def get_attributes_by_relation(self, object_relation): - '''Returns the list of attributes with the given object relation in the object''' - return self._fast_attribute_access.get(object_relation, []) - - @property - def _fast_attribute_access(self): - if not self.__fast_attribute_access: - for a in self.attributes: - self.__fast_attribute_access[a.object_relation].append(a) - return self.__fast_attribute_access - - def has_attributes_by_relation(self, list_of_relations): - '''True if all the relations in the list are defined in the object''' - return all(relation in self._fast_attribute_access for relation in list_of_relations) - - def add_attribute(self, object_relation, simple_value=None, **value): - """Add an attribute. object_relation is required and the value key is a - dictionary with all the keys supported by MISPAttribute""" - if simple_value is not None: # /!\ The value *can* be 0 - value = {'value': simple_value} - if value.get('value') in [None, '']: - logger.warning("The value of the attribute you're trying to add is None or empty string, skipping it. Object relation: {}".format(object_relation)) - return None - if self._known_template: - if object_relation in self._definition['attributes']: - attribute = MISPObjectAttribute(self._definition['attributes'][object_relation]) - else: - # Woopsie, this object_relation is unknown, no sane defaults for you. - logger.warning("The template ({}) doesn't have the object_relation ({}) you're trying to add.".format(self.name, object_relation)) - attribute = MISPObjectAttribute({}) - else: - attribute = MISPObjectAttribute({}) - # Overwrite the parameters of self._default_attributes_parameters with the ones of value - attribute.from_dict(object_relation=object_relation, **dict(self._default_attributes_parameters, **value)) - # FIXME New syntax python3 only, keep for later. - # attribute.from_dict(object_relation=object_relation, **{**self._default_attributes_parameters, **value}) - self.__fast_attribute_access[object_relation].append(attribute) - self.Attribute.append(attribute) - self.edited = True - return attribute - - def add_attributes(self, object_relation, *attributes): - '''Add multiple attributes with the same object_relation. - Helper for object_relation when multiple is True in the template. - It is the same as calling multiple times add_attribute with the same object_relation. - ''' - to_return = [] - for attribute in attributes: - if isinstance(attribute, dict): - a = self.add_attribute(object_relation, **attribute) - else: - a = self.add_attribute(object_relation, value=attribute) - to_return.append(a) - return to_return - - def to_dict(self, strict=False): - if strict or self._strict and self._known_template: - self._validate() - return super(MISPObject, self).to_dict() - - def to_json(self, strict=False, sort_keys=False, indent=None): - if strict or self._strict and self._known_template: - self._validate() - return super(MISPObject, self).to_json(sort_keys=sort_keys, indent=indent) - - def _validate(self): - """Make sure the object we're creating has the required fields""" - if self._definition.get('required'): - required_missing = set(self._definition.get('required')) - set(self._fast_attribute_access.keys()) - if required_missing: - raise InvalidMISPObject('{} are required.'.format(required_missing)) - if self._definition.get('requiredOneOf'): - if not set(self._definition['requiredOneOf']) & set(self._fast_attribute_access.keys()): - # We ecpect at least one of the object_relation in requiredOneOf, and it isn't the case - raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self._definition['requiredOneOf']))) - for rel, attrs in self._fast_attribute_access.items(): - if len(attrs) == 1: - # object_relation's here only once, everything's cool, moving on - continue - if not self._definition['attributes'][rel].get('multiple'): - # object_relation's here more than once, but it isn't allowed in the template. - raise InvalidMISPObject('Multiple occurrences of {} is not allowed'.format(rel)) - return True - - def __repr__(self): - if hasattr(self, 'name'): - return '<{self.__class__.__name__}(name={self.name})'.format(self=self) - return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) diff --git a/pymisp/tools/__init__.py b/pymisp/tools/__init__.py index 14a9ed0..4aba8ac 100644 --- a/pymisp/tools/__init__.py +++ b/pymisp/tools/__init__.py @@ -1,5 +1,3 @@ -import sys - from .vtreportobject import VTReportObject # noqa from .neo4j import Neo4j # noqa from .fileobject import FileObject # noqa @@ -16,14 +14,13 @@ from .domainipobject import DomainIPObject # noqa from .asnobject import ASNObject # noqa from .geolocationobject import GeolocationObject # noqa -if sys.version_info >= (3, 6): - from .emailobject import EMailObject # noqa - from .vehicleobject import VehicleObject # noqa - from .csvloader import CSVLoader # noqa - from .sshauthkeyobject import SSHAuthorizedKeysObject # noqa - from .feed import feed_meta_generator # noqa - try: - from .urlobject import URLObject # noqa - except ImportError: - # Requires faup, which is a bit difficult to install - pass +from .emailobject import EMailObject # noqa +from .vehicleobject import VehicleObject # noqa +from .csvloader import CSVLoader # noqa +from .sshauthkeyobject import SSHAuthorizedKeysObject # noqa +from .feed import feed_meta_generator # noqa +try: + from .urlobject import URLObject # noqa +except ImportError: + # Requires faup, which is a bit difficult to install + pass diff --git a/setup.py b/setup.py index 8704798..ccb5969 100644 --- a/setup.py +++ b/setup.py @@ -34,15 +34,12 @@ setup( 'Intended Audience :: Science/Research', 'Intended Audience :: Telecommunications Industry', 'Intended Audience :: Information Technology', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', 'Topic :: Security', 'Topic :: Internet', ], - install_requires=['six', 'requests', 'python-dateutil', 'jsonschema', - 'python-dateutil', 'enum34;python_version<"3.4"', - 'functools32;python_version<"3.0"', 'deprecated', 'cachetools;python_version<"3.0"'], - extras_require={'fileobjects': ['lief>=0.8,<0.10;python_version<"3.5"', 'lief>=0.10.1;python_version>"3.5"', 'python-magic', 'pydeep'], + install_requires=['six', 'requests', 'python-dateutil', 'jsonschema', 'deprecated'], + extras_require={'fileobjects': ['lief>=0.10.1', 'python-magic', 'pydeep'], 'neo': ['py2neo'], 'openioc': ['beautifulsoup4'], 'virustotal': ['validators'], diff --git a/tests/test.py b/tests/test.py deleted file mode 100755 index 45f55e4..0000000 --- a/tests/test.py +++ /dev/null @@ -1,314 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -from pymisp import PyMISP, __version__ -try: - from keys import url, key -except ImportError as e: - print(e) - url = 'https://localhost:8443' - key = 'd6OmdDFvU3Seau3UjwvHS1y3tFQbaRNhJhDX0tjh' - -import time - -import unittest - - -class TestBasic(unittest.TestCase): - - def setUp(self): - self.maxDiff = None - self.misp = PyMISP(url, key, False, 'json') - self.live_describe_types = self.misp.get_live_describe_types() - - def _clean_event(self, event): - event['Event'].pop('orgc_id', None) - event['Event'].pop('uuid', None) - event['Event'].pop('sharing_group_id', None) - event['Event'].pop('timestamp', None) - event['Event'].pop('org_id', None) - event['Event'].pop('date', None) - event['Event'].pop('RelatedEvent', None) - event['Event'].pop('publish_timestamp', None) - if event['Event'].get('Attribute'): - for a in event['Event'].get('Attribute'): - a.pop('uuid', None) - a.pop('event_id', None) - a.pop('id', None) - a.pop('timestamp', None) - if event['Event'].get('Orgc'): - event['Event']['Orgc'].pop('uuid', None) - event['Event']['Orgc'].pop('id', None) - if event['Event'].get('Org'): - event['Event']['Org'].pop('uuid', None) - event['Event']['Org'].pop('id', None) - return event['Event'].pop('id', None) - - def new_event(self): - event = self.misp.new_event(0, 1, 0, "This is a test") - event_id = self._clean_event(event) - to_check = {u'Event': {u'info': u'This is a test', u'locked': False, - u'attribute_count': u'0', 'disable_correlation': False, u'analysis': u'0', - u'ShadowAttribute': [], u'published': False, - u'distribution': u'0', u'event_creator_email': u'admin@admin.test', u'Attribute': [], u'proposal_email_lock': False, - u'extends_uuid': '', - u'Object': [], u'Org': {'local': True, u'name': u'ORGNAME'}, - u'Orgc': {'local': True, u'name': u'ORGNAME'}, - u'Galaxy': [], - u'threat_level_id': u'1'}} - self.assertEqual(event, to_check, 'Failed at creating a new Event') - return int(event_id) - - def add_hashes(self, eventid): - r = self.misp.get_event(eventid) - event = r.json() - event = self.misp.add_hashes(event, - category='Payload installation', - filename='dll_installer.dll', - md5='0a209ac0de4ac033f31d6ba9191a8f7a', - sha1='1f0ae54ac3f10d533013f74f48849de4e65817a7', - sha256='003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9', - ssdeep=None, - comment='Fanny modules', - to_ids=False, - distribution=2, - proposal=False) - self._clean_event(event) - to_check = {u'Event': {u'info': u'This is a test', u'locked': False, - u'attribute_count': u'3', u'analysis': u'0', - u'ShadowAttribute': [], u'published': False, u'distribution': u'0', u'event_creator_email': u'admin@admin.test', - u'Org': {'local': True, u'name': u'ORGNAME'}, - u'Orgc': {'local': True, u'name': u'ORGNAME'}, - u'Galaxy': [], - u'Attribute': [ - {u'category': u'Payload installation', u'comment': u'Fanny modules', - u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a', - u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5'}, - {u'category': u'Payload installation', u'comment': u'Fanny modules', - u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7', - u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1'}, - {u'category': u'Payload installation', u'comment': u'Fanny modules', - u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9', - u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}], - u'proposal_email_lock': False, u'threat_level_id': u'1'}} - self.assertEqual(event, to_check, 'Failed at adding hashes') - - def publish(self, eventid): - r = self.misp.get_event(eventid) - event = r.json() - event = self.misp.publish(event) - self._clean_event(event) - to_check = {u'Event': {u'info': u'This is a test', u'locked': False, - u'attribute_count': u'3', u'analysis': u'0', - u'ShadowAttribute': [], u'published': True, u'distribution': u'0', u'event_creator_email': u'admin@admin.test', - u'Org': {'local': True, u'name': u'ORGNAME'}, - u'Orgc': {'local': True, u'name': u'ORGNAME'}, - u'Galaxy': [], - u'Attribute': [ - {u'category': u'Payload installation', u'comment': u'Fanny modules', - u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a', - u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5'}, - {u'category': u'Payload installation', u'comment': u'Fanny modules', - u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7', - u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1'}, - {u'category': u'Payload installation', u'comment': u'Fanny modules', - u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9', - u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}], - u'proposal_email_lock': False, u'threat_level_id': u'1'}} - self.assertEqual(event, to_check, 'Failed at publishing event') - - def delete(self, eventid): - event = self.misp.delete_event(eventid) - print(event) - - def delete_attr(self, attrid): - event = self.misp.delete_attribute(attrid) - print(event) - - def get(self, eventid): - event = self.misp.get_event(eventid) - print(event) - - def get_stix(self, **kwargs): - event = self.misp.get_stix(kwargs) - print(event) - - def add(self): - event = {u'Event': {u'info': u'This is a test', u'locked': False, - u'attribute_count': u'3', u'analysis': u'0', - u'ShadowAttribute': [], u'published': False, u'distribution': u'0', u'event_creator_email': u'admin@admin.test', - u'Attribute': [ - {u'category': u'Payload installation', u'comment': u'Fanny modules', - u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a', - u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5'}, - {u'category': u'Payload installation', u'comment': u'Fanny modules', - u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7', - u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1'}, - {u'category': u'Payload installation', u'comment': u'Fanny modules', - u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9', - u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}], - u'proposal_email_lock': False, u'threat_level_id': u'1'}} - event = self.misp.add_event(event) - print(event) - - def add_user(self): - email = 'test@misp.local' - role_id = '5' - org_id = '1' - password = 'Password1234!' - external_auth_required = False - external_auth_key = '' - enable_password = False - nids_sid = '1238717' - server_id = '1' - gpgkey = '' - certif_public = '' - autoalert = False - contactalert = False - disabled = False - change_pw = '0' - termsaccepted = False - newsread = '0' - authkey = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' - to_check = {'User': {'email': email, 'org_id': org_id, 'role_id': role_id, - 'password': password, 'external_auth_required': external_auth_required, - 'external_auth_key': external_auth_key, 'enable_password': enable_password, - 'nids_sid': nids_sid, 'server_id': server_id, 'gpgkey': gpgkey, - 'certif_public': certif_public, 'autoalert': autoalert, - 'contactalert': contactalert, 'disabled': disabled, - 'change_pw': change_pw, 'termsaccepted': termsaccepted, - 'newsread': newsread, 'authkey': authkey}} - user = self.misp.add_user(email=email, - role_id=role_id, - org_id=org_id, - password=password, - external_auth_required=external_auth_required, - external_auth_key=external_auth_key, - enable_password=enable_password, - nids_sid=nids_sid, - server_id=server_id, - gpgkey=gpgkey, - certif_public=certif_public, - autoalert=autoalert, - contactalert=contactalert, - disabled=disabled, - change_pw=change_pw, - termsaccepted=termsaccepted, - newsread=newsread, - authkey=authkey) - # delete user to allow reuse of test - uid = user.get('User').get('id') - self.misp.delete_user(uid) - # ---------------------------------- - # test interesting keys only (some keys are modified(password) and some keys are added (lastlogin) - tested_keys = ['email', 'org_id', 'role_id', 'server_id', 'autoalert', - 'authkey', 'gpgkey', 'certif_public', 'nids_sid', 'termsaccepted', - 'newsread', 'contactalert', 'disabled'] - for k in tested_keys: - self.assertEqual(user.get('User').get(k), to_check.get('User').get(k), "Failed to match input with output on key: {}".format(k)) - - def add_organisation(self): - name = 'Organisation tests' - description = 'This is a test organisation' - orgtype = 'Type is a string' - nationality = 'French' - sector = 'Bank sector' - uuid = '16fd2706-8baf-433b-82eb-8c7fada847da' - contacts = 'Text field with no limitations' - local = False - to_check = {'Organisation': {'name': name, 'description': description, - 'type': orgtype, 'nationality': nationality, - 'sector': sector, 'uuid': uuid, 'contacts': contacts, - 'local': local}} - org = self.misp.add_organisation(name=name, - description=description, - type=orgtype, - nationality=nationality, - sector=sector, - uuid=uuid, - contacts=contacts, - local=local, - ) - # delete organisation to allow reuse of test - oid = org.get('Organisation').get('id') - self.misp.delete_organisation(oid) - # ---------------------------------- - tested_keys = ['anonymise', 'contacts', 'description', 'local', 'name', - 'nationality', 'sector', 'type', 'uuid'] - for k in tested_keys: - self.assertEqual(org.get('Organisation').get(k), to_check.get('Organisation').get(k), "Failed to match input with output on key: {}".format(k)) - - def test_create_event(self): - eventid = self.new_event() - time.sleep(1) - self.delete(eventid) - - def test_get_event(self): - eventid = self.new_event() - time.sleep(1) - self.get(eventid) - time.sleep(1) - self.delete(eventid) - - def test_add_event(self): - self.add() - time.sleep(1) - self.delete(1) - - def test_del_attr(self): - eventid = self.new_event() - time.sleep(1) - self.delete_attr(1) - time.sleep(1) - self.delete(eventid) - - def test_one_or_more(self): - self.assertEqual(self.misp._one_or_more(1), (1,)) - self.assertEqual(self.misp._one_or_more([1]), [1]) - - def test_create_user(self): - self.add_user() - - def test_create_organisation(self): - self.add_organisation() - - def test_describeTypes_sane_default(self): - sane_default = self.live_describe_types['sane_defaults'] - self.assertEqual(sorted(sane_default.keys()), sorted(self.live_describe_types['types'])) - - def test_describeTypes_categories(self): - category_type_mappings = self.live_describe_types['category_type_mappings'] - self.assertEqual(sorted(category_type_mappings.keys()), sorted(self.live_describe_types['categories'])) - - def test_describeTypes_types_in_categories(self): - category_type_mappings = self.live_describe_types['category_type_mappings'] - for category, types in category_type_mappings.items(): - existing_types = [t for t in types if t in self.live_describe_types['types']] - self.assertEqual(sorted(existing_types), sorted(types)) - - def test_describeTypes_types_have_category(self): - category_type_mappings = self.live_describe_types['category_type_mappings'] - all_types = set() - for category, types in category_type_mappings.items(): - all_types.update(types) - self.assertEqual(sorted(list(all_types)), sorted(self.live_describe_types['types'])) - - def test_describeTypes_sane_default_valid_category(self): - sane_default = self.live_describe_types['sane_defaults'] - categories = self.live_describe_types['categories'] - for t, sd in sane_default.items(): - self.assertTrue(sd['to_ids'] in [0, 1]) - self.assertTrue(sd['default_category'] in categories) - - def test_live_acl(self): - query_acl = self.misp.get_live_query_acl() - self.assertEqual(query_acl['response'], []) - - def test_recommended_pymisp_version(self): - response = self.misp.get_recommended_api_version() - recommended_version_tup = tuple(int(x) for x in response['version'].split('.')) - pymisp_version_tup = tuple(int(x) for x in __version__.split('.'))[:3] - self.assertEqual(recommended_version_tup, pymisp_version_tup) - - -if __name__ == '__main__': - unittest.main() diff --git a/travis/install_travis.sh b/travis/install_travis.sh index 268b128..079d11a 100644 --- a/travis/install_travis.sh +++ b/travis/install_travis.sh @@ -3,11 +3,6 @@ set -e set -x -if [ ${LEGACY} == true ]; then - pip install nose coveralls codecov requests-mock pydeep - pip install .[fileobjects] -else - # We're in python3, installing with pipenv. - pip install pipenv - pipenv update --dev -fi +# We're in python3, installing with pipenv. +pip install pipenv +pipenv update --dev diff --git a/travis/test_travis.sh b/travis/test_travis.sh index d773f95..8018fd4 100644 --- a/travis/test_travis.sh +++ b/travis/test_travis.sh @@ -3,9 +3,4 @@ set -e set -x -if [ -z ${LEGACY} ]; then - # We're in python3, test all and use pipenv. - pipenv run nosetests-3.4 --with-coverage --cover-package=pymisp,tests --cover-tests tests/test_*.py -else - nosetests --with-coverage --cover-package=pymisp,tests --cover-tests tests/test_mispevent.py -fi +pipenv run nosetests-3.4 --with-coverage --cover-package=pymisp,tests --cover-tests tests/test_*.py