diff --git a/Pipfile b/Pipfile index 53f5a09..3059c5a 100644 --- a/Pipfile +++ b/Pipfile @@ -3,11 +3,6 @@ name = "pypi" url = "https://pypi.org/simple" verify_ssl = true -[[source]] -name = "lief_index" -url = "https://lief-project.github.io/packages/" -verify_ssl = true - [dev-packages] nose = "*" coveralls = "*" @@ -17,7 +12,6 @@ requests-mock = "*" [packages] pymisp = {editable = true,extras = ["fileobjects", "neo", "openioc", "virustotal", "pdfexport"],path = "."} pymispwarninglists = {editable = true,git = "https://github.com/MISP/PyMISPWarningLists.git"} -lief = {version = ">=0.10.0.dev0",index = "lief_index",markers = "python_version >= '3.5'"} [requires] python_version = "3" diff --git a/Pipfile.lock b/Pipfile.lock index 638d517..ce880ea 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "4056bb4063c740e772370f0dc360c08ac4e45bdbee16d0717aa1ef2698c08653" + "sha256": "92d8e062fe9d5baadd6145057fd6bd30db2c696628a6b3d697ae66431f4dace0" }, "pipfile-spec": 6, "requires": { @@ -12,11 +12,6 @@ "name": "pypi", "url": "https://pypi.org/simple", "verify_ssl": true - }, - { - "name": "lief_index", - "url": "https://lief-project.github.io/packages/", - "verify_ssl": true } ] }, @@ -71,6 +66,13 @@ ], "version": "==4.4.0" }, + "deprecated": { + "hashes": [ + "sha256:a515c4cf75061552e0284d123c3066fbbe398952c87333a92b8fc3dd8e4f9cc1", + "sha256:b07b414c8aac88f60c1d837d21def7e83ba711052e03b3cbaff27972567a8f8d" + ], + "version": "==1.2.6" + }, "idna": { "hashes": [ "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", @@ -100,8 +102,6 @@ "sha256:efa5f3523c01f7f0f5f2c14e5ac808e2447d1435c6a2872e5ab1a97ef1b0db9b", "sha256:f1aadb344b5e14b308167bd2c9f31f1915e3c4e3f9a9ca92ff7b7bfbede5034c" ], - "index": "lief_index", - "markers": "python_version >= '3.5'", "version": "==0.10.0.dev0" }, "neobolt": { @@ -118,34 +118,34 @@ }, "pillow": { "hashes": [ - "sha256:15c056bfa284c30a7f265a41ac4cbbc93bdbfc0dfe0613b9cb8a8581b51a9e55", - "sha256:1a4e06ba4f74494ea0c58c24de2bb752818e9d504474ec95b0aa94f6b0a7e479", - "sha256:1c3c707c76be43c9e99cb7e3d5f1bee1c8e5be8b8a2a5eeee665efbf8ddde91a", - "sha256:1fd0b290203e3b0882d9605d807b03c0f47e3440f97824586c173eca0aadd99d", - "sha256:24114e4a6e1870c5a24b1da8f60d0ba77a0b4027907860188ea82bd3508c80eb", - "sha256:258d886a49b6b058cd7abb0ab4b2b85ce78669a857398e83e8b8e28b317b5abb", - "sha256:33c79b6dd6bc7f65079ab9ca5bebffb5f5d1141c689c9c6a7855776d1b09b7e8", - "sha256:367385fc797b2c31564c427430c7a8630db1a00bd040555dfc1d5c52e39fcd72", - "sha256:3c1884ff078fb8bf5f63d7d86921838b82ed4a7d0c027add773c2f38b3168754", - "sha256:44e5240e8f4f8861d748f2a58b3f04daadab5e22bfec896bf5434745f788f33f", - "sha256:46aa988e15f3ea72dddd81afe3839437b755fffddb5e173886f11460be909dce", - "sha256:74d90d499c9c736d52dd6d9b7221af5665b9c04f1767e35f5dd8694324bd4601", - "sha256:809c0a2ce9032cbcd7b5313f71af4bdc5c8c771cb86eb7559afd954cab82ebb5", - "sha256:85d1ef2cdafd5507c4221d201aaf62fc9276f8b0f71bd3933363e62a33abc734", - "sha256:8c3889c7681af77ecfa4431cd42a2885d093ecb811e81fbe5e203abc07e0995b", - "sha256:9218d81b9fca98d2c47d35d688a0cea0c42fd473159dfd5612dcb0483c63e40b", - "sha256:9aa4f3827992288edd37c9df345783a69ef58bd20cc02e64b36e44bcd157bbf1", - "sha256:9d80f44137a70b6f84c750d11019a3419f409c944526a95219bea0ac31f4dd91", - "sha256:b7ebd36128a2fe93991293f997e44be9286503c7530ace6a55b938b20be288d8", - "sha256:c4c78e2c71c257c136cdd43869fd3d5e34fc2162dc22e4a5406b0ebe86958239", - "sha256:c6a842537f887be1fe115d8abb5daa9bc8cc124e455ff995830cc785624a97af", - "sha256:cf0a2e040fdf5a6d95f4c286c6ef1df6b36c218b528c8a9158ec2452a804b9b8", - "sha256:cfd28aad6fc61f7a5d4ee556a997dc6e5555d9381d1390c00ecaf984d57e4232", - "sha256:dca5660e25932771460d4688ccbb515677caaf8595f3f3240ec16c117deff89a", - "sha256:de7aedc85918c2f887886442e50f52c1b93545606317956d65f342bd81cb4fc3", - "sha256:e6c0bbf8e277b74196e3140c35f9a1ae3eafd818f7f2d3a15819c49135d6c062" + "sha256:0804f77cb1e9b6dbd37601cee11283bba39a8d44b9ddb053400c58e0c0d7d9de", + "sha256:0ab7c5b5d04691bcbd570658667dd1e21ca311c62dcfd315ad2255b1cd37f64f", + "sha256:0b3e6cf3ea1f8cecd625f1420b931c83ce74f00c29a0ff1ce4385f99900ac7c4", + "sha256:365c06a45712cd723ec16fa4ceb32ce46ad201eb7bbf6d3c16b063c72b61a3ed", + "sha256:38301fbc0af865baa4752ddae1bb3cbb24b3d8f221bf2850aad96b243306fa03", + "sha256:3aef1af1a91798536bbab35d70d35750bd2884f0832c88aeb2499aa2d1ed4992", + "sha256:3fe0ab49537d9330c9bba7f16a5f8b02da615b5c809cdf7124f356a0f182eccd", + "sha256:45a619d5c1915957449264c81c008934452e3fd3604e36809212300b2a4dab68", + "sha256:49f90f147883a0c3778fd29d3eb169d56416f25758d0f66775db9184debc8010", + "sha256:571b5a758baf1cb6a04233fb23d6cf1ca60b31f9f641b1700bfaab1194020555", + "sha256:5ac381e8b1259925287ccc5a87d9cf6322a2dc88ae28a97fe3e196385288413f", + "sha256:6153db744a743c0c8c91b8e3b9d40e0b13a5d31dbf8a12748c6d9bfd3ddc01ad", + "sha256:6fd63afd14a16f5d6b408f623cc2142917a1f92855f0df997e09a49f0341be8a", + "sha256:70acbcaba2a638923c2d337e0edea210505708d7859b87c2bd81e8f9902ae826", + "sha256:70b1594d56ed32d56ed21a7fbb2a5c6fd7446cdb7b21e749c9791eac3a64d9e4", + "sha256:76638865c83b1bb33bcac2a61ce4d13c17dba2204969dedb9ab60ef62bede686", + "sha256:7b2ec162c87fc496aa568258ac88631a2ce0acfe681a9af40842fc55deaedc99", + "sha256:7cee2cef07c8d76894ebefc54e4bb707dfc7f258ad155bd61d87f6cd487a70ff", + "sha256:7d16d4498f8b374fc625c4037742fbdd7f9ac383fd50b06f4df00c81ef60e829", + "sha256:b50bc1780681b127e28f0075dfb81d6135c3a293e0c1d0211133c75e2179b6c0", + "sha256:bd0582f831ad5bcad6ca001deba4568573a4675437db17c4031939156ff339fa", + "sha256:cfd40d8a4b59f7567620410f966bb1f32dc555b2b19f82a91b147fac296f645c", + "sha256:e3ae410089de680e8f84c68b755b42bc42c0ceb8c03dbea88a5099747091d38e", + "sha256:e9046e559c299b395b39ac7dbf16005308821c2f24a63cae2ab173bd6aa11616", + "sha256:ef6be704ae2bc8ad0ebc5cb850ee9139493b0fc4e81abcc240fb392a63ebc808", + "sha256:f8dc19d92896558f9c4317ee365729ead9d7bbcf2052a9a19a3ef17abbb8ac5b" ], - "version": "==6.0.0" + "version": "==6.1.0" }, "prompt-toolkit": { "hashes": [ @@ -192,9 +192,9 @@ }, "pyrsistent": { "hashes": [ - "sha256:16692ee739d42cf5e39cef8d27649a8c1fdb7aa99887098f1460057c5eb75c3a" + "sha256:50cffebc87ca91b9d4be2dcc2e479272bcb466b5a0487b6c271f7ddea6917e14" ], - "version": "==0.15.2" + "version": "==0.15.3" }, "python-dateutil": { "hashes": [ @@ -290,6 +290,12 @@ "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c" ], "version": "==0.1.7" + }, + "wrapt": { + "hashes": [ + "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1" + ], + "version": "==1.11.2" } }, "develop": { diff --git a/examples/generate_file_objects.py b/examples/generate_file_objects.py index 3269845..c3eda36 100755 --- a/examples/generate_file_objects.py +++ b/examples/generate_file_objects.py @@ -5,7 +5,7 @@ import argparse import json try: - from pymisp import MISPEncode + from pymisp import MISPEncode, AbstractMISP from pymisp.tools import make_binary_objects except ImportError: pass @@ -59,6 +59,7 @@ if __name__ == '__main__': group.add_argument("-p", "--path", help="Path to process.") group.add_argument("-c", "--check", action='store_true', help="Check the dependencies.") args = parser.parse_args() + a = AbstractMISP() if args.check: print(check()) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 32accde..7f7f66e 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,6 +1,5 @@ __version__ = '2.4.111' import logging -import functools import warnings import sys @@ -14,28 +13,26 @@ logger.addHandler(default_handler) logger.setLevel(logging.WARNING) -def deprecated(func): - '''This is a decorator which can be used to mark functions - as deprecated. It will result in a warning being emitted - when the function is used.''' +def warning_2020(): - @functools.wraps(func) - def new_func(*args, **kwargs): - warnings.showwarning( - "Call to deprecated function {}.".format(func.__name__), - category=DeprecationWarning, - filename=func.__code__.co_filename, - lineno=func.__code__.co_firstlineno + 1 - ) - return func(*args, **kwargs) - return new_func + if sys.version_info < (3, 6): + warnings.warn(""" +Python 2.7 is officially end of life the 2020-01-01. For this occasion, +we decided to review which versions of Python we support and our conclusion +is to only support python 3.6+ starting the 2020-01-01. + +Every version of pymisp released after the 2020-01-01 will fail if the +python interpreter is prior to python 3.6. + +**Please update your codebase.**""", DeprecationWarning, stacklevel=3) try: + warning_2020() from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate, PyMISPInvalidFormat, MISPServerError, PyMISPNotImplementedYet, PyMISPUnexpectedResponse, PyMISPEmptyResponse # noqa from .api import PyMISP # noqa from .abstract import AbstractMISP, MISPEncode, MISPTag, Distribution, ThreatLevel, Analysis # noqa - from .mispevent import MISPEvent, MISPAttribute, MISPObjectReference, MISPObjectAttribute, MISPObject, MISPUser, MISPOrganisation, MISPSighting, MISPLog, MISPShadowAttribute # noqa + from .mispevent import MISPEvent, MISPAttribute, MISPObjectReference, MISPObjectAttribute, MISPObject, MISPUser, MISPOrganisation, MISPSighting, MISPLog, MISPShadowAttribute, MISPWarninglist, MISPTaxonomy, MISPNoticelist, MISPObjectTemplate, MISPSharingGroup, MISPRole, MISPServer, MISPFeed # noqa from .tools import AbstractMISPObjectGenerator # noqa from .tools import Neo4j # noqa from .tools import stix # noqa @@ -44,6 +41,7 @@ try: from .tools import ext_lookups # noqa if sys.version_info >= (3, 6): + from .aping import ExpandedPyMISP # noqa # Let's not bother with old python try: from .tools import reportlab_generator # noqa @@ -53,8 +51,6 @@ try: except NameError: # FIXME: The import should not raise an exception if reportlab isn't installed pass - if sys.version_info >= (3, 6): - from .aping import ExpandedPyMISP # noqa logger.debug('pymisp loaded properly') except ImportError as e: logger.warning('Unable to load pymisp properly: {}'.format(e)) diff --git a/pymisp/abstract.py b/pymisp/abstract.py index f8cdf7d..8261acf 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -13,14 +13,13 @@ from .exceptions import PyMISPInvalidFormat # Try to import MutableMapping the python 3.3+ way try: from collections.abc import MutableMapping -except: +except Exception: pass logger = logging.getLogger('pymisp') if sys.version_info < (3, 0): - logger.warning("You're using python 2, it is strongly recommended to use python >=3.6") from collections import MutableMapping # This is required because Python 2 is a pain. @@ -74,7 +73,7 @@ class MISPEncode(JSONEncoder): def default(self, obj): if isinstance(obj, AbstractMISP): return obj.jsonable() - elif isinstance(obj, datetime.datetime): + elif isinstance(obj, (datetime.datetime, datetime.date)): return obj.isoformat() elif isinstance(obj, Enum): return obj.value @@ -270,16 +269,17 @@ class AbstractMISP(MutableMapping): else: return False + def __repr__(self): + if hasattr(self, 'name'): + return '<{self.__class__.__name__}(name={self.name})'.format(self=self) + return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) + class MISPTag(AbstractMISP): def __init__(self): super(MISPTag, self).__init__() - def from_dict(self, name, **kwargs): - self.name = name + def from_dict(self, **kwargs): + if kwargs.get('Tag'): + kwargs = kwargs.get('Tag') super(MISPTag, self).from_dict(**kwargs) - - def __repr__(self): - if hasattr(self, 'name'): - return '<{self.__class__.__name__}(name={self.name})'.format(self=self) - return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) diff --git a/pymisp/api.py b/pymisp/api.py index 90411f3..e898882 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -14,8 +14,9 @@ import re import logging from io import BytesIO, open import zipfile +from deprecated import deprecated -from . import __version__, deprecated +from . import __version__, warning_2020 from .exceptions import PyMISPError, SearchError, NoURL, NoKey, PyMISPEmptyResponse from .mispevent import MISPEvent, MISPAttribute, MISPUser, MISPOrganisation, MISPSighting, MISPFeed, MISPObject, MISPSharingGroup from .abstract import AbstractMISP, MISPEncode @@ -29,7 +30,6 @@ try: unicode = str except ImportError: from urlparse import urljoin - logger.warning("You're using python 2, it is strongly recommended to use python >=3.6") try: import requests @@ -58,12 +58,12 @@ Response (if any): {}''' -class PyMISP(object): +class PyMISP(object): # pragma: no cover """Python API for MISP :param url: URL of the MISP instance you want to connect to :param key: API key of the user you want to use - :param ssl: can be True or False (to check ot not the validity of the certificate. Or a CA_BUNDLE in case of self signed certificate (the concatenation of all the \*.crt of the chain) + :param ssl: can be True or False (to check ot not the validity of the certificate. Or a CA_BUNDLE in case of self signed certificate (the concatenation of all the *.crt of the chain) :param out_type: Type of object (json) NOTE: XML output isn't supported anymore, keeping the flag for compatibility reasons. :param debug: Write all the debug information to stderr :param proxies: Proxy dict as describes here: http://docs.python-requests.org/en/master/user/advanced/#proxies @@ -73,6 +73,9 @@ class PyMISP(object): :param tool: The software using PyMISP (string), used to set a unique user-agent """ + warning_2020() + + @deprecated(reason="Please use ExpandedPyMISP instead (requires Python 3.6+). This class will be removed an alias of ExpandedPyMISP early 2020.") def __init__(self, url, key, ssl=True, out_type='json', debug=None, proxies=None, cert=None, asynch=False, auth=None, tool=None): if not url: raise NoURL('Please provide the URL of your MISP instance.') @@ -129,11 +132,13 @@ class PyMISP(object): def __repr__(self): return '<{self.__class__.__name__}(url={self.root_url})'.format(self=self) + @deprecated(reason="Use ExpandedPyMISP.remote_acl", version='2.4.111') def get_live_query_acl(self): """This should return an empty list, unless the ACL is outdated.""" response = self._prepare_request('GET', urljoin(self.root_url, 'events/queryACL.json')) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.describe_types_local", version='2.4.110') def get_local_describe_types(self): with open(os.path.join(self.resources_path, 'describeTypes.json'), 'rb') as f: if OLD_PY3: @@ -142,6 +147,7 @@ class PyMISP(object): describe_types = json.load(f) return describe_types['result'] + @deprecated(reason="Use ExpandedPyMISP.describe_types_remote", version='2.4.110') def get_live_describe_types(self): response = self._prepare_request('GET', urljoin(self.root_url, 'attributes/describeTypes.json')) describe_types = self._check_response(response) @@ -328,6 +334,7 @@ class PyMISP(object): # ############### Simple REST API ################ # ################################################ + @deprecated(reason="Not used, open an issue if required.", version='2.4.110') def test_connection(self): """Test the auth key""" response = self.get_version() @@ -335,6 +342,7 @@ class PyMISP(object): raise PyMISPError(response.get('errors')[0]) return True + @deprecated(reason="Use ExpandedPyMISP.search_index") def get_index(self, filters=None): """Return the index. @@ -347,6 +355,7 @@ class PyMISP(object): response = self._prepare_request('POST', url, json.dumps(filters)) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.get_event") def get_event(self, event_id): """Get an event @@ -356,6 +365,7 @@ class PyMISP(object): response = self._prepare_request('GET', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.get_object") def get_object(self, obj_id): """Get an object @@ -365,6 +375,7 @@ class PyMISP(object): response = self._prepare_request('GET', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.get_attribute") def get_attribute(self, att_id): """Get an attribute @@ -374,6 +385,7 @@ class PyMISP(object): response = self._prepare_request('GET', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.add_event") def add_event(self, event): """Add a new event @@ -387,6 +399,7 @@ class PyMISP(object): response = self._prepare_request('POST', url, event) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.update_attribute") def update_attribute(self, attribute_id, attribute): """Update an attribute @@ -401,6 +414,7 @@ class PyMISP(object): response = self._prepare_request('POST', url, attribute) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.update_event") def update_event(self, event_id, event): """Update an event @@ -415,6 +429,7 @@ class PyMISP(object): response = self._prepare_request('POST', url, event) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.delete_event") def delete_event(self, event_id): """Delete an event @@ -424,6 +439,7 @@ class PyMISP(object): response = self._prepare_request('DELETE', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.delete_attribute") def delete_attribute(self, attribute_id, hard_delete=False): """Delete an attribute by ID""" if hard_delete: @@ -433,12 +449,14 @@ class PyMISP(object): response = self._prepare_request('GET', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.push_event_to_ZMQ") def pushEventToZMQ(self, event_id): """Force push an event on ZMQ""" url = urljoin(self.root_url, 'events/pushEventToZMQ/{}.json'.format(event_id)) response = self._prepare_request('POST', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.direct_call") def direct_call(self, url, data=None): '''Very lightweight call that posts a data blob (python dictionary or json string) on the URL''' url = urljoin(self.root_url, url) @@ -454,10 +472,12 @@ class PyMISP(object): # ############### Event handling ############### # ############################################## + @deprecated(reason="Use ExpandedPyMISP.get_event") def get(self, eid): """Get an event by event ID""" return self.get_event(eid) + @deprecated(reason="Use ExpandedPyMISP.update_event") def update(self, event): """Update an event by ID""" e = self._make_mispevent(event) @@ -467,6 +487,7 @@ class PyMISP(object): eid = e.id return self.update_event(eid, e) + @deprecated(reason="Use ExpandedPyMISP.publish") def fast_publish(self, event_id, alert=False): """Does the same as the publish method, but just try to publish the event even with one single HTTP GET. @@ -479,6 +500,7 @@ class PyMISP(object): response = self._prepare_request('POST', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.publish") def publish(self, event, alert=True): """Publish event (with or without alert email) :param event: pass event or event id (as string or int) to publish @@ -494,38 +516,44 @@ class PyMISP(object): event_id = full_event.id return self.fast_publish(event_id, alert) + @deprecated(reason="Use ExpandedPyMISP.update_event") def change_threat_level(self, event, threat_level_id): """Change the threat level of an event""" e = self._make_mispevent(event) e.threat_level_id = threat_level_id return self.update(e) + @deprecated(reason="Use ExpandedPyMISP.update_event") def change_analysis_status(self, event, analysis_status): """Change the analysis status of an event""" e = self._make_mispevent(event) e.analysis = analysis_status return self.update(e) + @deprecated(reason="Use ExpandedPyMISP.update_event") def change_distribution(self, event, distribution): """Change the distribution of an event""" e = self._make_mispevent(event) e.distribution = distribution return self.update(e) + @deprecated(reason="Use ExpandedPyMISP.update_sharing_group") def change_sharing_group(self, event, sharing_group_id): """Change the sharing group of an event""" e = self._make_mispevent(event) e.distribution = 4 # Needs to be 'Sharing group' if e.SharingGroup: # Delete former SharingGroup information del e.SharingGroup - e.sharing_group_id = sharing_group_id # Set new sharing group id + e.sharing_group_id = sharing_group_id # Set new sharing group id return self.update(e) + @deprecated(reason="Use ExpandedPyMISP.add_event") def new_event(self, distribution=None, threat_level_id=None, analysis=None, info=None, date=None, published=False, orgc_id=None, org_id=None, sharing_group_id=None): """Create and add a new event""" misp_event = self._prepare_full_event(distribution, threat_level_id, analysis, info, date, published, orgc_id, org_id, sharing_group_id) return self.add_event(misp_event) + @deprecated(reason="Use ExpandedPyMISP.tag", version='2.4.111') def tag(self, uuid, tag): """Tag an event or an attribute""" if not self._valid_uuid(uuid): @@ -535,6 +563,7 @@ class PyMISP(object): response = self._prepare_request('POST', url, json.dumps(to_post)) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.untag", version='2.4.111') def untag(self, uuid, tag): """Untag an event or an attribute""" if not self._valid_uuid(uuid): @@ -621,6 +650,7 @@ class PyMISP(object): event_id = e['uuid'] return event_id + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_named_attribute(self, event, type_value, value, category=None, to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): """Add one or more attributes to an existing event""" attributes = [] @@ -628,6 +658,7 @@ class PyMISP(object): attributes.append(self._prepare_full_attribute(category, type_value, value, to_ids, comment, distribution, **kwargs)) return self._send_attributes(event, attributes, proposal) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_hashes(self, event, category='Artifacts dropped', filename=None, md5=None, sha1=None, sha256=None, ssdeep=None, comment=None, to_ids=True, distribution=None, proposal=False, **kwargs): """Add hashe(s) to an existing event""" @@ -648,18 +679,22 @@ class PyMISP(object): return self._send_attributes(event, attributes, proposal) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def av_detection_link(self, event, link, category='Antivirus detection', to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): """Add AV detection link(s)""" return self.add_named_attribute(event, 'link', link, category, to_ids, comment, distribution, proposal, **kwargs) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_detection_name(self, event, name, category='Antivirus detection', to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): """Add AV detection name(s)""" return self.add_named_attribute(event, 'text', name, category, to_ids, comment, distribution, proposal, **kwargs) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_filename(self, event, filename, category='Artifacts dropped', to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): """Add filename(s)""" return self.add_named_attribute(event, 'filename', filename, category, to_ids, comment, distribution, proposal, **kwargs) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_attachment(self, event, attachment, category='Artifacts dropped', to_ids=False, comment=None, distribution=None, proposal=False, filename=None, **kwargs): """Add an attachment to the MISP event @@ -704,6 +739,7 @@ class PyMISP(object): # Send it on its way return self.add_named_attribute(event, 'attachment', filename, category, to_ids, comment, distribution, proposal, data=encodedData, **kwargs) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_regkey(self, event, regkey, rvalue=None, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): """Add a registry key""" if rvalue: @@ -717,6 +753,7 @@ class PyMISP(object): attributes.append(self._prepare_full_attribute(category, type_value, value, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_regkeys(self, event, regkeys_values, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): """Add a registry keys""" attributes = [] @@ -732,6 +769,7 @@ class PyMISP(object): attributes.append(self._prepare_full_attribute(category, type_value, value, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_pattern(self, event, pattern, in_file=True, in_memory=False, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): """Add a pattern(s) in file or in memory""" if not (in_file or in_memory): @@ -739,6 +777,7 @@ class PyMISP(object): itemtype = 'pattern-in-file' if in_file else 'pattern-in-memory' return self.add_named_attribute(event, itemtype, pattern, category, to_ids, comment, distribution, proposal, **kwargs) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_pipe(self, event, named_pipe, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): """Add pipes(s)""" def scrub(s): @@ -748,6 +787,7 @@ class PyMISP(object): attributes = list(map(scrub, self._one_or_more(named_pipe))) return self.add_named_attribute(event, 'named pipe', attributes, category, to_ids, comment, distribution, proposal, **kwargs) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_mutex(self, event, mutex, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): """Add mutex(es)""" def scrub(s): @@ -757,28 +797,34 @@ class PyMISP(object): attributes = list(map(scrub, self._one_or_more(mutex))) return self.add_named_attribute(event, 'mutex', attributes, category, to_ids, comment, distribution, proposal, **kwargs) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_yara(self, event, yara, category='Payload delivery', to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): """Add yara rule(es)""" return self.add_named_attribute(event, 'yara', yara, category, to_ids, comment, distribution, proposal, **kwargs) # ##### Network attributes ##### + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_ipdst(self, event, ipdst, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): """Add destination IP(s)""" return self.add_named_attribute(event, 'ip-dst', ipdst, category, to_ids, comment, distribution, proposal, **kwargs) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_ipsrc(self, event, ipsrc, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): """Add source IP(s)""" return self.add_named_attribute(event, 'ip-src', ipsrc, category, to_ids, comment, distribution, proposal, **kwargs) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_hostname(self, event, hostname, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): """Add hostname(s)""" return self.add_named_attribute(event, 'hostname', hostname, category, to_ids, comment, distribution, proposal, **kwargs) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_domain(self, event, domain, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): """Add domain(s)""" return self.add_named_attribute(event, 'domain', domain, category, to_ids, comment, distribution, proposal, **kwargs) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_domain_ip(self, event, domain, ip, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): """Add domain|ip""" if isinstance(ip, str): @@ -786,117 +832,143 @@ class PyMISP(object): composed = list(map(lambda x: '%s|%s' % (domain, x), ip)) return self.add_named_attribute(event, 'domain|ip', composed, category, to_ids, comment, distribution, proposal, **kwargs) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_domains_ips(self, event, domain_ips, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): """Add multiple domain|ip""" composed = list(map(lambda x: '%s|%s' % (x[0], x[1]), domain_ips.items())) return self.add_named_attribute(event, 'domain|ip', composed, category, to_ids, comment, distribution, proposal, **kwargs) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_url(self, event, url, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): """Add url(s)""" return self.add_named_attribute(event, 'url', url, category, to_ids, comment, distribution, proposal, **kwargs) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_useragent(self, event, useragent, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): """Add user agent(s)""" return self.add_named_attribute(event, 'user-agent', useragent, category, to_ids, comment, distribution, proposal, **kwargs) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_traffic_pattern(self, event, pattern, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): """Add pattern(s) in traffic""" return self.add_named_attribute(event, 'pattern-in-traffic', pattern, category, to_ids, comment, distribution, proposal, **kwargs) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_snort(self, event, snort, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): """Add SNORT rule(s)""" return self.add_named_attribute(event, 'snort', snort, category, to_ids, comment, distribution, proposal, **kwargs) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_asn(self, event, asn, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): """Add network ASN""" return self.add_named_attribute(event, 'AS', asn, category, to_ids, comment, distribution, proposal, **kwargs) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_net_other(self, event, netother, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): """Add a free text entry""" return self.add_named_attribute(event, 'other', netother, category, to_ids, comment, distribution, proposal, **kwargs) # ##### Email attributes ##### + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_email_src(self, event, email, category='Payload delivery', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): """Add a source email""" return self.add_named_attribute(event, 'email-src', email, category, to_ids, comment, distribution, proposal, **kwargs) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_email_dst(self, event, email, category='Payload delivery', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): """Add a destination email""" return self.add_named_attribute(event, 'email-dst', email, category, to_ids, comment, distribution, proposal, **kwargs) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_email_subject(self, event, email, category='Payload delivery', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): """Add an email subject""" return self.add_named_attribute(event, 'email-subject', email, category, to_ids, comment, distribution, proposal, **kwargs) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_email_attachment(self, event, email, category='Payload delivery', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): """Add an email atachment""" return self.add_named_attribute(event, 'email-attachment', email, category, to_ids, comment, distribution, proposal, **kwargs) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_email_header(self, event, email, category='Payload delivery', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): """Add an email header""" return self.add_named_attribute(event, 'email-header', email, category, to_ids, comment, distribution, proposal, **kwargs) # ##### Target attributes ##### + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_target_email(self, event, target, category='Targeting data', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): """Add an target email""" return self.add_named_attribute(event, 'target-email', target, category, to_ids, comment, distribution, proposal, **kwargs) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_target_user(self, event, target, category='Targeting data', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): """Add an target user""" return self.add_named_attribute(event, 'target-user', target, category, to_ids, comment, distribution, proposal, **kwargs) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_target_machine(self, event, target, category='Targeting data', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): """Add an target machine""" return self.add_named_attribute(event, 'target-machine', target, category, to_ids, comment, distribution, proposal, **kwargs) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_target_org(self, event, target, category='Targeting data', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): """Add an target organisation""" return self.add_named_attribute(event, 'target-org', target, category, to_ids, comment, distribution, proposal, **kwargs) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_target_location(self, event, target, category='Targeting data', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): """Add an target location""" return self.add_named_attribute(event, 'target-location', target, category, to_ids, comment, distribution, proposal, **kwargs) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_target_external(self, event, target, category='Targeting data', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): """Add an target external""" return self.add_named_attribute(event, 'target-external', target, category, to_ids, comment, distribution, proposal, **kwargs) # ##### Attribution attributes ##### + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_threat_actor(self, event, target, category='Attribution', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): """Add an threat actor""" return self.add_named_attribute(event, 'threat-actor', target, category, to_ids, comment, distribution, proposal, **kwargs) # ##### Internal reference attributes ##### + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_internal_link(self, event, reference, category='Internal reference', to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): """Add an internal link""" return self.add_named_attribute(event, 'link', reference, category, to_ids, comment, distribution, proposal, **kwargs) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_internal_comment(self, event, reference, category='Internal reference', to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): """Add an internal comment""" return self.add_named_attribute(event, 'comment', reference, category, to_ids, comment, distribution, proposal, **kwargs) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_internal_text(self, event, reference, category='Internal reference', to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): """Add an internal text""" return self.add_named_attribute(event, 'text', reference, category, to_ids, comment, distribution, proposal, **kwargs) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_internal_other(self, event, reference, category='Internal reference', to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): """Add an internal reference (type other)""" return self.add_named_attribute(event, 'other', reference, category, to_ids, comment, distribution, proposal, **kwargs) # ##### Other attributes ##### + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_other_comment(self, event, reference, category='Other', to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): """Add other comment""" return self.add_named_attribute(event, 'comment', reference, category, to_ids, comment, distribution, proposal, **kwargs) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_other_counter(self, event, reference, category='Other', to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): """Add other counter""" return self.add_named_attribute(event, 'counter', reference, category, to_ids, comment, distribution, proposal, **kwargs) + @deprecated(reason="Use ExpandedPyMISP.add_attribute and MISPAttribute") def add_other_text(self, event, reference, category='Other', to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): """Add other text""" return self.add_named_attribute(event, 'text', reference, category, to_ids, comment, distribution, proposal, **kwargs) @@ -1001,6 +1073,7 @@ class PyMISP(object): response = self._prepare_request('POST', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.get_attribute_proposal", version='2.4.111') def proposal_view(self, event_id=None, proposal_id=None): """View a proposal""" if proposal_id is not None and event_id is not None: @@ -1011,18 +1084,22 @@ class PyMISP(object): id = proposal_id return self.__query_proposal('view', id) + @deprecated(reason="Use ExpandedPyMISP.add_attribute_proposal", version='2.4.111') def proposal_add(self, event_id, attribute): """Add a proposal""" return self.__query_proposal('add', event_id, attribute) + @deprecated(reason="Use ExpandedPyMISP.update_attribute_proposal", version='2.4.111') def proposal_edit(self, attribute_id, attribute): """Edit a proposal""" return self.__query_proposal('edit', attribute_id, attribute) + @deprecated(reason="Use ExpandedPyMISP.accept_attribute_proposal", version='2.4.111') def proposal_accept(self, proposal_id): """Accept a proposal""" return self.__query_proposal('accept', proposal_id) + @deprecated(reason="Use ExpandedPyMISP.discard_attribute_proposal", version='2.4.111') def proposal_discard(self, proposal_id): """Discard a proposal""" return self.__query_proposal('discard', proposal_id) @@ -1031,6 +1108,7 @@ class PyMISP(object): # ###### Attribute update ###### # ############################## + @deprecated(reason="Use ExpandedPyMISP.update_attribute and MISPAttribute") def change_toids(self, attribute_uuid, to_ids): """Change the toids flag""" if to_ids not in [0, 1]: @@ -1038,11 +1116,13 @@ class PyMISP(object): query = {"to_ids": to_ids} return self.__query('edit/{}'.format(attribute_uuid), query, controller='attributes') + @deprecated(reason="Use ExpandedPyMISP.update_attribute and MISPAttribute") def change_comment(self, attribute_uuid, comment): """Change the comment of attribute""" query = {"comment": comment} return self.__query('edit/{}'.format(attribute_uuid), query, controller='attributes') + @deprecated(reason="Use ExpandedPyMISP.update_attribute and MISPAttribute") def change_disable_correlation(self, attribute_uuid, disable_correlation): """Change the disable_correlation flag""" possible_values = [0, 1, False, True] @@ -1055,6 +1135,7 @@ class PyMISP(object): # ###### Attribute update ###### # ############################## + @deprecated(reason="Use ExpandedPyMISP.freetext", version='2.4.111') def freetext(self, event_id, string, adhereToWarninglists=False, distribution=None, returnMetaAttributes=False): """Pass a text to the freetext importer""" query = {"value": string} @@ -1087,6 +1168,7 @@ class PyMISP(object): response = self._prepare_request('POST', url, json.dumps(query)) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.search_index", version='2.4.111') def search_index(self, published=None, eventid=None, tag=None, datefrom=None, dateuntil=None, eventinfo=None, threatlevel=None, distribution=None, analysis=None, attribute=None, org=None, async_callback=None, normalize=False, @@ -1150,6 +1232,7 @@ class PyMISP(object): res = to_return return res + @deprecated(reason="Use ExpandedPyMISP.search", version='2.4.111') def search_all(self, value): """Search a value in the whole database""" query = {'value': value, 'searchall': 1} @@ -1174,6 +1257,7 @@ class PyMISP(object): to_return.append('!{}'.format(not_values)) return to_return + @deprecated(reason="Use ExpandedPyMISP.search", version='2.4.111') def search(self, controller='events', async_callback=None, **kwargs): """Search via the Rest API @@ -1279,6 +1363,7 @@ class PyMISP(object): # Create a session, make it async if and only if we have a callback return self.__query('restSearch', query, controller, async_callback) + @deprecated(reason="Use ExpandedPyMISP.get_attribute", version='2.4.111') def get_attachment(self, attribute_id): """Get an attachement (not a malware sample) by attribute ID. Returns the attachment as a bytestream, or a dictionary containing the error message. @@ -1295,6 +1380,7 @@ class PyMISP(object): # content contains the attachment in binary return response.content + @deprecated(reason="Use ExpandedPyMISP.search", version='2.4.111') def get_yara(self, event_id): """Get the yara rules from an event""" url = urljoin(self.root_url, 'attributes/restSearch') @@ -1308,6 +1394,7 @@ class PyMISP(object): rules = '\n\n'.join([a['value'] for a in result['response']['Attribute']]) return True, rules + @deprecated(reason="Use ExpandedPyMISP.search. Open an issue if needed.", version='2.4.111') def download_samples(self, sample_hash=None, event_id=None, all_samples=False, unzip=True): """Download samples, by hash or event ID. If there are multiple samples in one event, use the all_samples switch @@ -1348,6 +1435,7 @@ class PyMISP(object): details.append([f['event_id'], "{0}.zip".format(f['filename']), zipped]) return True, details + @deprecated(reason="Use ExpandedPyMISP.search", version='2.4.111') def download_last(self, last): """Download the last published events. @@ -1366,6 +1454,7 @@ class PyMISP(object): timestamp = (pydate - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds() return timestamp + @deprecated(reason="Use ExpandedPyMISP.search", version='2.4.111') def get_events_last_modified(self, search_from, search_to=None): """Download the last modified events. @@ -1385,6 +1474,7 @@ class PyMISP(object): # ########## Tags ########## + @deprecated(reason="Use ExpandedPyMISP.tags", version='2.4.111') def get_all_tags(self, quiet=False): """Get all the tags used on the instance""" url = urljoin(self.root_url, 'tags') @@ -1398,6 +1488,7 @@ class PyMISP(object): to_return.append(tag['name']) return to_return + @deprecated(reason="Use ExpandedPyMISP.add_tag", version='2.4.111') def new_tag(self, name=None, colour="#00ace6", exportable=False, hide_tag=False): """Create a new tag""" to_post = {'Tag': {'name': name, 'colour': colour, 'exportable': exportable, 'hide_tag': hide_tag}} @@ -1407,10 +1498,12 @@ class PyMISP(object): # ########## Version ########## + @deprecated(reason="Use ExpandedPyMISP.version", version='2.4.110') def get_api_version(self): """Returns the current version of PyMISP installed on the system""" return {'version': __version__} + @deprecated(reason="Use ExpandedPyMISP.pymisp_version_master", version='2.4.110') def get_api_version_master(self): """Get the most recent version of PyMISP from github""" r = requests.get('https://raw.githubusercontent.com/MISP/PyMISP/master/pymisp/__init__.py') @@ -1420,18 +1513,21 @@ class PyMISP(object): else: return {'error': 'Impossible to retrieve the version of the master branch.'} + @deprecated(reason="Use ExpandedPyMISP.recommended_pymisp_version", version='2.4.110') def get_recommended_api_version(self): """Returns the recommended API version from the server""" url = urljoin(self.root_url, 'servers/getPyMISPVersion.json') response = self._prepare_request('GET', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.misp_instance_version", version='2.4.110') def get_version(self): """Returns the version of the instance.""" url = urljoin(self.root_url, 'servers/getVersion.json') response = self._prepare_request('GET', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.misp_instance_version_master", version='2.4.110') def get_version_master(self): """Get the most recent version from github""" r = requests.get('https://raw.githubusercontent.com/MISP/MISP/2.4/VERSION.json') @@ -1443,6 +1539,7 @@ class PyMISP(object): # ############## Statistics ################## + @deprecated(reason="Use ExpandedPyMISP.attributes_statistics", version='2.4.110') def get_attributes_statistics(self, context='type', percentage=None): """Get attributes statistics from the MISP instance""" if (context != 'category'): @@ -1454,6 +1551,7 @@ class PyMISP(object): response = self._prepare_request('GET', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.tags_statistics", version='2.4.110') def get_tags_statistics(self, percentage=None, name_sort=None): """Get tags statistics from the MISP instance""" if percentage is not None: @@ -1468,6 +1566,7 @@ class PyMISP(object): response = self._prepare_request('GET', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.users_statistics", version='2.4.110') def get_users_statistics(self, context='data'): """Get users statistics from the MISP instance""" availables_contexts = ['data', 'orgs', 'users', 'tags', 'attributehistogram', 'sightings', 'attackMatrix'] @@ -1479,18 +1578,21 @@ class PyMISP(object): # ############## Sightings ################## + @deprecated(reason="Use ExpandedPyMISP.add_sighting", version='2.4.110') def sighting_per_id(self, attribute_id): """Add a sighting to an attribute (by attribute ID)""" url = urljoin(self.root_url, 'sightings/add/{}'.format(attribute_id)) response = self._prepare_request('POST', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.add_sighting", version='2.4.110') def sighting_per_uuid(self, attribute_uuid): """Add a sighting to an attribute (by attribute UUID)""" url = urljoin(self.root_url, 'sightings/add/{}'.format(attribute_uuid)) response = self._prepare_request('POST', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.add_sighting", version='2.4.110') def set_sightings(self, sightings): """Push a sighting (python dictionary or MISPSighting) or a list of sightings""" if not isinstance(sightings, list): @@ -1504,12 +1606,14 @@ class PyMISP(object): response = self._prepare_request('POST', url, to_post) return self._check_response(response) + @deprecated(reason="Not used, open an issue if required.", version='2.4.110') def sighting_per_json(self, json_file): """Push a sighting (JSON file)""" with open(json_file, 'rb') as f: jdata = json.load(f) return self.set_sightings(jdata) + @deprecated(reason="Use ExpandedPyMISP.add_sighting", version='2.4.110') def sighting(self, value=None, uuid=None, id=None, source=None, type=None, timestamp=None, **kwargs): """ Set a single sighting. :value: Value of the attribute the sighting is related too. Pushing this object @@ -1524,6 +1628,7 @@ class PyMISP(object): s.from_dict(value=value, uuid=uuid, id=id, source=source, type=type, timestamp=timestamp, **kwargs) return self.set_sightings(s) + @deprecated(reason="Use ExpandedPyMISP.sightings", version='2.4.110') def sighting_list(self, element_id, scope="attribute", org_id=False): """Get the list of sighting. :param element_id: could be an event id or attribute id @@ -1554,6 +1659,7 @@ class PyMISP(object): response = self._prepare_request('POST', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.search_sightings", version='2.4.110') def search_sightings(self, context='', async_callback=None, **kwargs): """Search sightings via the REST API :context: The context of the search, could be attribute, event or False @@ -1603,6 +1709,7 @@ class PyMISP(object): # ############## Sharing Groups ################## + @deprecated(reason="Use ExpandedPyMISP.sharing_groups", version='2.4.110') def get_sharing_groups(self): """Get the existing sharing groups""" url = urljoin(self.root_url, 'sharing_groups.json') @@ -1611,12 +1718,15 @@ class PyMISP(object): # ############## Users ################## + @deprecated(reason="Use ExpandedPyMISP.users", version='2.4.110') def get_users_list(self): return self._rest_list('admin/users') + @deprecated(reason="Use ExpandedPyMISP.get_user", version='2.4.110') def get_user(self, user_id='me'): return self._rest_view('users', user_id) + @deprecated(reason="Use ExpandedPyMISP.add_user", version='2.4.110') def add_user(self, email, org_id=None, role_id=None, **kwargs): if isinstance(email, MISPUser): # Very dirty, allow to call that from ExpandedPyMISP @@ -1626,6 +1736,7 @@ class PyMISP(object): new_user.from_dict(email=email, org_id=org_id, role_id=role_id, **kwargs) return self._rest_add('admin/users', new_user) + @deprecated(reason="Not used, open an issue if required.", version='2.4.110') def add_user_json(self, json_file): with open(json_file, 'rb') as f: jdata = json.load(f) @@ -1633,14 +1744,17 @@ class PyMISP(object): new_user.from_dict(**jdata) return self._rest_add('admin/users', new_user) + @deprecated(reason="Not used, open an issue if required.", version='2.4.110') def get_user_fields_list(self): return self._rest_get_parameters('admin/users') + @deprecated(reason="Use ExpandedPyMISP.update_user", version='2.4.110') def edit_user(self, user_id, **kwargs): edit_user = MISPUser() edit_user.from_dict(**kwargs) return self._rest_edit('admin/users', edit_user, user_id) + @deprecated(reason="Not used, open an issue if required.", version='2.4.110') def edit_user_json(self, json_file, user_id): with open(json_file, 'rb') as f: jdata = json.load(f) @@ -1648,20 +1762,24 @@ class PyMISP(object): new_user.from_dict(**jdata) return self._rest_edit('admin/users', new_user, user_id) + @deprecated(reason="Use ExpandedPyMISP.delete_user", version='2.4.110') def delete_user(self, user_id): return self._rest_delete('admin/users', user_id) # ############## Organisations ################## + @deprecated(reason="Use ExpandedPyMISP.organisations", version='2.4.110') def get_organisations_list(self, scope="local"): scope = scope.lower() if scope not in ["local", "external", "all"]: raise ValueError("Authorized fields are 'local','external' or 'all'") return self._rest_list('organisations/index/scope:{}'.format(scope)) + @deprecated(reason="Use ExpandedPyMISP.get_organisation", version='2.4.110') def get_organisation(self, organisation_id): return self._rest_view('organisations', organisation_id) + @deprecated(reason="Use ExpandedPyMISP.add_organisation", version='2.4.110') def add_organisation(self, name, **kwargs): if isinstance(name, MISPOrganisation): # Very dirty, allow to call that from ExpandedPyMISP @@ -1675,6 +1793,7 @@ class PyMISP(object): raise PyMISPError('A remote org MUST have a valid uuid') return self._rest_add('admin/organisations', new_org) + @deprecated(reason="Not used, open an issue if required.", version='2.4.110') def add_organisation_json(self, json_file): with open(json_file, 'rb') as f: jdata = json.load(f) @@ -1682,14 +1801,17 @@ class PyMISP(object): new_org.from_dict(**jdata) return self._rest_add('admin/organisations', new_org) + @deprecated(reason="Not used, open an issue if required.", version='2.4.110') def get_organisation_fields_list(self): return self._rest_get_parameters('admin/organisations') + @deprecated(reason="Use ExpandedPyMISP.update_organisation", version='2.4.110') def edit_organisation(self, org_id, **kwargs): edit_org = MISPOrganisation() edit_org.from_dict(**kwargs) return self._rest_edit('admin/organisations', edit_org, org_id) + @deprecated(reason="Not used, open an issue if required.", version='2.4.110') def edit_organisation_json(self, json_file, org_id): with open(json_file, 'rb') as f: jdata = json.load(f) @@ -1697,6 +1819,7 @@ class PyMISP(object): edit_org.from_dict(**jdata) return self._rest_edit('admin/organisations', edit_org, org_id) + @deprecated(reason="Use ExpandedPyMISP.delete_organisation", version='2.4.110') def delete_organisation(self, org_id): return self._rest_delete('admin/organisations', org_id) @@ -1754,6 +1877,7 @@ class PyMISP(object): server['delete_client_cert'] = delete_client_cert return server + @deprecated(reason="Use ExpandedPyMISP.add_server", version='2.4.110') def add_server(self, url, name, authkey, organisation, internal=None, push=False, pull=False, self_signed=False, push_rules="", pull_rules="", submitted_cert=None, submitted_client_cert=None): @@ -1764,6 +1888,7 @@ class PyMISP(object): response = self._prepare_request('POST', url, json.dumps(new_server)) return self._check_response(response) + @deprecated(reason="Not used, open an issue if required.", version='2.4.110') def add_server_json(self, json_file): with open(json_file, 'rb') as f: jdata = json.load(f) @@ -1771,6 +1896,7 @@ class PyMISP(object): response = self._prepare_request('POST', url, json.dumps(jdata)) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.update_server", version='2.4.110') def edit_server(self, server_id, url=None, name=None, authkey=None, organisation=None, internal=None, push=False, pull=False, self_signed=False, push_rules="", pull_rules="", submitted_cert=None, submitted_client_cert=None, delete_cert=None, delete_client_cert=None): @@ -1781,6 +1907,7 @@ class PyMISP(object): response = self._prepare_request('POST', url, json.dumps(new_server)) return self._check_response(response) + @deprecated(reason="Not used, open an issue if required.", version='2.4.110') def edit_server_json(self, json_file, server_id): with open(json_file, 'rb') as f: jdata = json.load(f) @@ -1788,6 +1915,7 @@ class PyMISP(object): response = self._prepare_request('POST', url, json.dumps(jdata)) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.server_pull", version='2.4.110') def server_pull(self, server_id, event_id=None): url = urljoin(self.root_url, 'servers/pull/{}'.format(server_id)) if event_id is not None: @@ -1795,6 +1923,7 @@ class PyMISP(object): response = self._prepare_request('GET', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.server_push", version='2.4.110') def server_push(self, server_id, event_id=None): url = urljoin(self.root_url, 'servers/push/{}'.format(server_id)) if event_id is not None: @@ -1802,6 +1931,7 @@ class PyMISP(object): response = self._prepare_request('GET', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.servers", version='2.4.110') def servers_index(self): url = urljoin(self.root_url, 'servers/index') response = self._prepare_request('GET', url) @@ -1809,6 +1939,7 @@ class PyMISP(object): # ############## Roles ################## + @deprecated(reason="Use ExpandedPyMISP.roles", version='2.4.110') def get_roles_list(self): """Get the list of existing roles""" url = urljoin(self.root_url, 'roles') @@ -1817,12 +1948,14 @@ class PyMISP(object): # ############## Tags ################## + @deprecated(reason="Use ExpandedPyMISP.tags", version='2.4.110') def get_tags_list(self): """Get the list of existing tags.""" url = urljoin(self.root_url, 'tags') response = self._prepare_request('GET', url) return self._check_response(response)['Tag'] + @deprecated(reason="Use ExpandedPyMISP.get_tag", version='2.4.110') def get_tag(self, tag_id): """Get a tag by id.""" url = urljoin(self.root_url, 'tags/view/{}'.format(tag_id)) @@ -1853,6 +1986,7 @@ class PyMISP(object): return {'Tag': tag} + @deprecated(reason="Use ExpandedPyMISP.update_tag", version='2.4.110') def edit_tag(self, tag_id, name=None, colour=None, exportable=None, hide_tag=None, org_id=None, count=None, user_id=None, numerical_value=None, attribute_count=None): """Edit only the provided parameters of a tag.""" @@ -1863,6 +1997,7 @@ class PyMISP(object): response = self._prepare_request('POST', url, json.dumps(new_tag)) return self._check_response(response) + @deprecated(reason="Not used, open an issue if required.", version='2.4.110') def edit_tag_json(self, json_file, tag_id): """Edit the tag using a json file.""" with open(json_file, 'rb') as f: @@ -1871,11 +2006,13 @@ class PyMISP(object): response = self._prepare_request('POST', url, json.dumps(jdata)) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.enable_tag", version='2.4.110') def enable_tag(self, tag_id): """Enable a tag by id.""" response = self.edit_tag(tag_id, hide_tag=False) return response + @deprecated(reason="Use ExpandedPyMISP.disable_tag", version='2.4.110') def disable_tag(self, tag_id): """Disable a tag by id.""" response = self.edit_tag(tag_id, hide_tag=True) @@ -1883,30 +2020,35 @@ class PyMISP(object): # ############## Taxonomies ################## + @deprecated(reason="Use ExpandedPyMISP.taxonomies", version='2.4.110') def get_taxonomies_list(self): """Get all the taxonomies.""" url = urljoin(self.root_url, 'taxonomies') response = self._prepare_request('GET', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.get_taxonomy", version='2.4.110') def get_taxonomy(self, taxonomy_id): """Get a taxonomy by id.""" url = urljoin(self.root_url, 'taxonomies/view/{}'.format(taxonomy_id)) response = self._prepare_request('GET', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.update_taxonomies", version='2.4.110') def update_taxonomies(self): """Update all the taxonomies.""" url = urljoin(self.root_url, 'taxonomies/update') response = self._prepare_request('POST', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.enable_taxonomy", version='2.4.110') def enable_taxonomy(self, taxonomy_id): """Enable a taxonomy by id.""" url = urljoin(self.root_url, 'taxonomies/enable/{}'.format(taxonomy_id)) response = self._prepare_request('POST', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.disable_taxonomy", version='2.4.110') def disable_taxonomy(self, taxonomy_id): """Disable a taxonomy by id.""" self.disable_taxonomy_tags(taxonomy_id) @@ -1914,12 +2056,14 @@ class PyMISP(object): response = self._prepare_request('POST', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.get_taxonomy", version='2.4.110') def get_taxonomy_tags_list(self, taxonomy_id): """Get all the tags of a taxonomy by id.""" url = urljoin(self.root_url, 'taxonomies/view/{}'.format(taxonomy_id)) response = self._prepare_request('GET', url) return self._check_response(response)["entries"] + @deprecated(reason="Use ExpandedPyMISP.enable_taxonomy_tags", version='2.4.110') def enable_taxonomy_tags(self, taxonomy_id): """Enable all the tags of a taxonomy by id.""" enabled = self.get_taxonomy(taxonomy_id)['Taxonomy']['enabled'] @@ -1928,6 +2072,7 @@ class PyMISP(object): response = self._prepare_request('POST', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.disable_taxonomy_tags", version='2.4.110') def disable_taxonomy_tags(self, taxonomy_id): """Disable all the tags of a taxonomy by id.""" url = urljoin(self.root_url, 'taxonomies/disableTag/{}'.format(taxonomy_id)) @@ -1936,24 +2081,28 @@ class PyMISP(object): # ############## WarningLists ################## + @deprecated(reason="Use ExpandedPyMISP.warninglists", version='2.4.110') def get_warninglists(self): """Get all the warninglists.""" url = urljoin(self.root_url, 'warninglists') response = self._prepare_request('GET', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.get_warninglist", version='2.4.110') def get_warninglist(self, warninglist_id): """Get a warninglist by id.""" url = urljoin(self.root_url, 'warninglists/view/{}'.format(warninglist_id)) response = self._prepare_request('GET', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.update_warninglists", version='2.4.110') def update_warninglists(self): """Update all the warninglists.""" url = urljoin(self.root_url, 'warninglists/update') response = self._prepare_request('POST', url) return self._check_response(response) + @deprecated(reason="Please ExpandedPyMISP.toggle_warninglist instead.") def toggle_warninglist(self, warninglist_id=None, warninglist_name=None, force_enable=None): '''Toggle (enable/disable) the status of a warninglist by ID. :param warninglist_id: ID of the WarningList @@ -1976,14 +2125,17 @@ class PyMISP(object): response = self._prepare_request('POST', url, json.dumps(query)) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.enable_warninglist", version='2.4.110') def enable_warninglist(self, warninglist_id): """Enable a warninglist by id.""" return self.toggle_warninglist(warninglist_id=warninglist_id, force_enable=True) + @deprecated(reason="Use ExpandedPyMISP.disable_warninglist", version='2.4.110') def disable_warninglist(self, warninglist_id): """Disable a warninglist by id.""" return self.toggle_warninglist(warninglist_id=warninglist_id, force_enable=False) + @deprecated(reason='Use ExpandedPyMISP.values_in_warninglist', version='2.4.110') def check_warninglist(self, value): """Check if IOC values are in warninglist""" url = urljoin(self.root_url, 'warninglists/checkValue') @@ -1992,30 +2144,35 @@ class PyMISP(object): # ############## NoticeLists ################## + @deprecated(reason="Use ExpandedPyMISP.noticelists", version='2.4.110') def get_noticelists(self): """Get all the noticelists.""" url = urljoin(self.root_url, 'noticelists') response = self._prepare_request('GET', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.get_noticelist", version='2.4.110') def get_noticelist(self, noticelist_id): """Get a noticelist by id.""" url = urljoin(self.root_url, 'noticelists/view/{}'.format(noticelist_id)) response = self._prepare_request('GET', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.update_noticelists", version='2.4.110') def update_noticelists(self): """Update all the noticelists.""" url = urljoin(self.root_url, 'noticelists/update') response = self._prepare_request('POST', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.enable_noticelist", version='2.4.110') def enable_noticelist(self, noticelist_id): """Enable a noticelist by id.""" url = urljoin(self.root_url, 'noticelists/enableNoticelist/{}/true'.format(noticelist_id)) response = self._prepare_request('POST', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.disable_noticelist", version='2.4.110') def disable_noticelist(self, noticelist_id): """Disable a noticelist by id.""" url = urljoin(self.root_url, 'noticelists/enableNoticelist/{}'.format(noticelist_id)) @@ -2024,18 +2181,21 @@ class PyMISP(object): # ############## Galaxies/Clusters ################## + @deprecated(reason="Use ExpandedPyMISP.galaxies", version='2.4.110') def get_galaxies(self): """Get all the galaxies.""" url = urljoin(self.root_url, 'galaxies') response = self._prepare_request('GET', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.get_galaxy", version='2.4.110') def get_galaxy(self, galaxy_id): """Get a galaxy by id.""" url = urljoin(self.root_url, 'galaxies/view/{}'.format(galaxy_id)) response = self._prepare_request('GET', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.update_galaxies", version='2.4.110') def update_galaxies(self): """Update all the galaxies.""" url = urljoin(self.root_url, 'galaxies/update') @@ -2048,12 +2208,14 @@ class PyMISP(object): # ############## Suricata ############## + @deprecated(reason="Use ExpandedPyMISP.search", version='2.4.111') def download_all_suricata(self): """Download all suricata rules events.""" url = urljoin(self.root_url, 'events/nids/suricata/download') response = self._prepare_request('GET', url, output_type='rules') return response + @deprecated(reason="Use ExpandedPyMISP.search", version='2.4.111') def download_suricata_rule_event(self, event_id): """Download one suricata rule event. @@ -2065,6 +2227,7 @@ class PyMISP(object): # ############## Text ############### + @deprecated(reason="Use ExpandedPyMISP.search", version='2.4.111') def get_all_attributes_txt(self, type_attr, tags=False, eventId=False, allowNonIDS=False, date_from=False, date_to=False, last=False, enforceWarninglist=False, allowNotPublished=False): """Get all attributes from a specific type as plain text. Only published and IDS flagged attributes are exported, except if stated otherwise.""" url = urljoin(self.root_url, 'attributes/text/download/%s/%s/%s/%s/%s/%s/%s/%s/%s' % (type_attr, tags, eventId, allowNonIDS, date_from, date_to, last, enforceWarninglist, allowNotPublished)) @@ -2073,6 +2236,7 @@ class PyMISP(object): # ############## STIX ############## + @deprecated(reason="Use ExpandedPyMISP.search", version='2.4.111') def get_stix_event(self, event_id=None, with_attachments=False, from_date=False, to_date=False, tags=False): """Get an event/events in STIX format""" if tags: @@ -2084,9 +2248,11 @@ class PyMISP(object): response = self._prepare_request('GET', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.search", version='2.4.111') def get_stix(self, **kwargs): return self.get_stix_event(**kwargs) + @deprecated(reason="Use ExpandedPyMISP.search", version='2.4.111') def get_csv(self, eventid=None, attributes=[], object_attributes=[], misp_types=[], context=False, ignore=False, last=None): """Get MISP values in CSV format :param eventid: The event ID to query @@ -2159,14 +2325,17 @@ class PyMISP(object): # ######## Feed ######### # ########################### + @deprecated(reason="Use ExpandedPyMISP.feeds instead") def get_feeds_list(self): """Get the content of all the feeds""" return self._rest_list('feeds') + @deprecated(reason="Use ExpandedPyMISP.get_feed instead") def get_feed(self, feed_id): """Get the content of a single feed""" return self._rest_view('feeds', feed_id) + @deprecated(reason="Use ExpandedPyMISP.add_feed instead") def add_feed(self, source_format, url, name, input_source, provider, **kwargs): """Delete a feed""" new_feed = MISPFeed() @@ -2174,73 +2343,79 @@ class PyMISP(object): input_source=input_source, provider=provider) return self._rest_add('feeds', new_feed) + @deprecated(reason="Not used, open an issue if required.", version='2.4.110') def get_feed_fields_list(self): return self._rest_get_parameters('feeds') + @deprecated(reason="Use ExpandedPyMISP.update_feed instead") def edit_feed(self, feed_id, **kwargs): """Delete a feed""" edit_feed = MISPFeed() edit_feed.from_dict(**kwargs) return self._rest_edit('feeds', edit_feed) + @deprecated(reason="Use ExpandedPyMISP.delete_feed instead") def delete_feed(self, feed_id): """Delete a feed""" return self._rest_delete('feeds', feed_id) + @deprecated(reason="Use ExpandedPyMISP.fetch_feed instead") def fetch_feed(self, feed_id): """Fetch one single feed""" url = urljoin(self.root_url, 'feeds/fetchFromFeed/{}'.format(feed_id)) response = self._prepare_request('GET', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.cache_all_feeds instead") def cache_feeds_all(self): """ Cache all the feeds""" url = urljoin(self.root_url, 'feeds/cacheFeeds/all') response = self._prepare_request('GET', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.cache_feeds instead") def cache_feed(self, feed_id): """Cache a specific feed""" url = urljoin(self.root_url, 'feeds/cacheFeeds/{}'.format(feed_id)) response = self._prepare_request('GET', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.cache_freetext_feeds instead") def cache_feeds_freetext(self): """Cache all the freetext feeds""" url = urljoin(self.root_url, 'feeds/cacheFeeds/freetext') response = self._prepare_request('GET', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.cache_misp_feeds instead") def cache_feeds_misp(self): """Cache all the MISP feeds""" url = urljoin(self.root_url, 'feeds/cacheFeeds/misp') response = self._prepare_request('GET', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.compare_feeds instead") def compare_feeds(self): """Generate the comparison matrix for all the MISP feeds""" url = urljoin(self.root_url, 'feeds/compareFeeds') response = self._prepare_request('GET', url) return self._check_response(response) - @deprecated + @deprecated(reason="Use ExpandedPyMISP.get_feed instead") def view_feed(self, feed_ids): """Alias for get_feed""" return self.get_feed(feed_ids) - @deprecated + @deprecated(reason="Use ExpandedPyMISP.feeds instead") def view_feeds(self): """Alias for get_feeds_list""" return self.get_feeds_list() - @deprecated - def cache_all_feeds(self): - """Alias for cache_feeds_all""" - return self.cache_feeds_all() - # ###################### # ### Sharing Groups ### # ###################### + + @deprecated(reason="Use ExpandedPyMISP.add_sharing_group", version='2.4.111') def add_sharing_group(self, name, releasability, description, active=True): """Add a new sharing group, which includes the organisation associated with the API key and the local server @@ -2256,6 +2431,7 @@ class PyMISP(object): description=description, active=active) return self._rest_add('sharing_groups', new_sg) + @deprecated(reason="Use ExpandedPyMISP.add_org_to_sharing_group", version='2.4.111') def sharing_group_org_add(self, sharing_group, organisation, extend=False): '''Add an organisation to a sharing group. :sharing_group: Sharing group's local instance ID, or Sharing group's global UUID @@ -2267,6 +2443,7 @@ class PyMISP(object): response = self._prepare_request('POST', url, json.dumps(to_jsonify)) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.remove_org_from_sharing_group", version='2.4.111') def sharing_group_org_remove(self, sharing_group, organisation): '''Remove an organisation from a sharing group. :sharing_group: Sharing group's local instance ID, or Sharing group's global UUID @@ -2277,6 +2454,7 @@ class PyMISP(object): response = self._prepare_request('POST', url, json.dumps(to_jsonify)) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.add_server_to_sharing_group", version='2.4.111') def sharing_group_server_add(self, sharing_group, server, all_orgs=False): '''Add a server to a sharing group. :sharing_group: Sharing group's local instance ID, or Sharing group's global UUID @@ -2288,6 +2466,7 @@ class PyMISP(object): response = self._prepare_request('POST', url, json.dumps(to_jsonify)) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.remove_server_from_sharing_group", version='2.4.111') def sharing_group_server_remove(self, sharing_group, server): '''Remove a server from a sharing group. :sharing_group: Sharing group's local instance ID, or Sharing group's global UUID @@ -2298,6 +2477,7 @@ class PyMISP(object): response = self._prepare_request('POST', url, json.dumps(to_jsonify)) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.delete_sharing_group", version='2.4.111') def delete_sharing_group(self, sharing_group): """Delete a sharing group :sharing_group: Sharing group's local instance ID, or Sharing group's global uuid @@ -2308,6 +2488,7 @@ class PyMISP(object): # ### Objects ### # ################### + @deprecated(reason="Use ExpandedPyMISP.add_object", version='2.4.111') def add_object(self, event_id, *args, **kwargs): """Add an object :param event_id: Event ID of the event to attach the object to @@ -2331,6 +2512,7 @@ class PyMISP(object): response = self._prepare_request('POST', url, misp_object.to_json()) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.update_object", version='2.4.111') def edit_object(self, misp_object, object_id=None): """Edit an existing object""" if object_id: @@ -2345,30 +2527,35 @@ class PyMISP(object): response = self._prepare_request('POST', url, misp_object.to_json()) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.delete_object", version='2.4.111') def delete_object(self, id): """Deletes an object""" url = urljoin(self.root_url, 'objects/delete/{}'.format(id)) response = self._prepare_request('POST', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.add_object_reference", version='2.4.111') def add_object_reference(self, misp_object_reference): """Add a reference to an object""" url = urljoin(self.root_url, 'object_references/add') response = self._prepare_request('POST', url, misp_object_reference.to_json()) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.delete_object_reference", version='2.4.111') def delete_object_reference(self, id): """Deletes a reference to an object""" url = urljoin(self.root_url, 'object_references/delete/{}'.format(id)) response = self._prepare_request('POST', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.object_templates", version='2.4.111') def get_object_templates_list(self): """Returns the list of Object templates available on the MISP instance""" url = urljoin(self.root_url, 'objectTemplates') response = self._prepare_request('GET', url) return self._check_response(response) + @deprecated(reason="Use ExpandedPyMISP.get_object_template", version='2.4.111') def get_object_template(self, object_uuid): """Gets the full object template corresponting the UUID passed as parameter""" url = urljoin(self.root_url, 'objectTemplates/view/{}'.format(object_uuid)) @@ -2378,6 +2565,7 @@ class PyMISP(object): return response['ObjectTemplate']['id'] return response + @deprecated(reason="Use ExpandedPyMISP.get_object_template - open an issue if you really need this method.", version='2.4.111') def get_object_template_id(self, object_uuid): """Gets the template ID corresponting the UUID passed as parameter""" template = self.get_object_template(object_uuid) @@ -2386,6 +2574,7 @@ class PyMISP(object): # Contains the error message. return template + @deprecated(reason="Use ExpandedPyMISP.update_object_templates", version='2.4.111') def update_object_templates(self): url = urljoin(self.root_url, 'objectTemplates/update') response = self._prepare_request('POST', url) @@ -2395,7 +2584,7 @@ class PyMISP(object): # ####### Deprecated ######## # ########################### - @deprecated + @deprecated(reason="Use ExpandedPyMISP.tag", version='2.4.111') def add_tag(self, event, tag, attribute=False): if attribute: to_post = {'request': {'Attribute': {'id': event['id'], 'tag': tag}}} @@ -2410,7 +2599,7 @@ class PyMISP(object): response = self._prepare_request('POST', url, json.dumps(to_post)) return self._check_response(response) - @deprecated + @deprecated(reason="Use ExpandedPyMISP.untag", version='2.4.111') def remove_tag(self, event, tag, attribute=False): if attribute: to_post = {'request': {'Attribute': {'id': event['id'], 'tag': tag}}} diff --git a/pymisp/aping.py b/pymisp/aping.py index 6c6af8a..0c549e9 100644 --- a/pymisp/aping.py +++ b/pymisp/aping.py @@ -1,16 +1,22 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from .exceptions import MISPServerError, PyMISPUnexpectedResponse, PyMISPNotImplementedYet -from .api import PyMISP, everything_broken -from .mispevent import MISPEvent, MISPAttribute, MISPSighting, MISPLog, MISPObject, MISPUser, MISPOrganisation, MISPShadowAttribute from typing import TypeVar, Optional, Tuple, List, Dict, Union from datetime import date, datetime import csv from pathlib import Path - import logging from urllib.parse import urljoin +import json +import requests +from requests.auth import AuthBase +import re + +from . import __version__ +from .exceptions import MISPServerError, PyMISPUnexpectedResponse, PyMISPNotImplementedYet, PyMISPError, NoURL, NoKey +from .api import everything_broken, PyMISP +from .mispevent import MISPEvent, MISPAttribute, MISPSighting, MISPLog, MISPObject, MISPUser, MISPOrganisation, MISPShadowAttribute, MISPWarninglist, MISPTaxonomy, MISPGalaxy, MISPNoticelist, MISPObjectReference, MISPObjectTemplate, MISPSharingGroup, MISPRole, MISPServer, MISPFeed +from .abstract import MISPEncode, MISPTag, AbstractMISP SearchType = TypeVar('SearchType', str, int) # str: string to search / list: values to search (OR) / dict: {'OR': [list], 'NOT': [list], 'AND': [list]} @@ -24,248 +30,1168 @@ logger = logging.getLogger('pymisp') class ExpandedPyMISP(PyMISP): + """Python API for MISP - def build_complex_query(self, or_parameters: Optional[List[SearchType]]=None, - and_parameters: Optional[List[SearchType]]=None, - not_parameters: Optional[List[SearchType]]=None): - to_return = {} - if and_parameters: - to_return['AND'] = and_parameters - if not_parameters: - to_return['NOT'] = not_parameters - if or_parameters: - to_return['OR'] = or_parameters + :param url: URL of the MISP instance you want to connect to + :param key: API key of the user you want to use + :param ssl: can be True or False (to check ot not the validity of the certificate. Or a CA_BUNDLE in case of self signed certificate (the concatenation of all the *.crt of the chain) + :param debug: Write all the debug information to stderr + :param proxies: Proxy dict as describes here: http://docs.python-requests.org/en/master/user/advanced/#proxies + :param cert: Client certificate, as described there: http://docs.python-requests.org/en/master/user/advanced/#client-side-certificates + :param auth: The auth parameter is passed directly to requests, as described here: http://docs.python-requests.org/en/master/user/authentication/ + :param tool: The software using PyMISP (string), used to set a unique user-agent + """ + + def __init__(self, url: str, key: str, ssl=True, debug: bool=False, proxies: dict={}, + cert: Tuple[str, tuple]=None, auth: AuthBase=None, tool: str=''): + if not url: + raise NoURL('Please provide the URL of your MISP instance.') + if not key: + raise NoKey('Please provide your authorization key.') + + self.root_url = url + self.key = key + self.ssl = ssl + self.proxies = proxies + self.cert = cert + self.auth = auth + self.tool = tool + + self.resources_path = Path(__file__).parent / 'data' + if debug: + logger.setLevel(logging.DEBUG) + logger.info('To configure logging in your script, leave it to None and use the following: import logging; logging.getLogger(\'pymisp\').setLevel(logging.DEBUG)') + + try: + # Make sure the MISP instance is working and the URL is valid + response = self.recommended_pymisp_version + if response.get('errors'): + logger.warning(response.get('errors')[0]) + else: + pymisp_version_tup = tuple(int(x) for x in __version__.split('.')) + recommended_version_tup = tuple(int(x) for x in response['version'].split('.')) + if recommended_version_tup < pymisp_version_tup[:3]: + logger.info(f"The version of PyMISP recommended by the MISP instance (response['version']) is older than the one you're using now ({__version__}). If you have a problem, please upgrade the MISP instance or use an older PyMISP version.") + elif pymisp_version_tup[:3] < recommended_version_tup: + logger.warning(f"The version of PyMISP recommended by the MI)SP instance ({response['version']}) is newer than the one you're using now ({__version__}). Please upgrade PyMISP.") + + except Exception as e: + raise PyMISPError(f'Unable to connect to MISP ({self.root_url}). Please make sure the API key and the URL are correct (http/https is required): {e}') + + try: + self.describe_types = self.describe_types_remote + except Exception: + self.describe_types = self.describe_types_local + + self.categories = self.describe_types['categories'] + self.types = self.describe_types['types'] + self.category_type_mapping = self.describe_types['category_type_mappings'] + self.sane_default = self.describe_types['sane_defaults'] + + @property + def remote_acl(self): + """This should return an empty list, unless the ACL is outdated.""" + response = self._prepare_request('GET', 'events/queryACL.json') + return self._check_response(response, expect_json=True) + + @property + def describe_types_local(self): + '''Returns the content of describe types from the package''' + with (self.resources_path / 'describeTypes.json').open() as f: + describe_types = json.load(f) + return describe_types['result'] + + @property + def describe_types_remote(self): + '''Returns the content of describe types from the remote instance''' + response = self._prepare_request('GET', 'attributes/describeTypes.json') + describe_types = self._check_response(response, expect_json=True) + return describe_types['result'] + + @property + def recommended_pymisp_version(self): + """Returns the recommended API version from the server""" + response = self._prepare_request('GET', 'servers/getPyMISPVersion.json') + return self._check_response(response, expect_json=True) + + @property + def version(self): + """Returns the version of PyMISP you're curently using""" + return {'version': __version__} + + @property + def pymisp_version_master(self): + """Get the most recent version of PyMISP from github""" + r = requests.get('https://raw.githubusercontent.com/MISP/PyMISP/master/pymisp/__init__.py') + if r.status_code == 200: + version = re.findall("__version__ = '(.*)'", r.text) + return {'version': version[0]} + return {'error': 'Impossible to retrieve the version of the master branch.'} + + @property + def misp_instance_version(self): + """Returns the version of the instance.""" + response = self._prepare_request('GET', 'servers/getVersion.json') + return self._check_response(response, expect_json=True) + + @property + def misp_instance_version_master(self): + """Get the most recent version from github""" + r = requests.get('https://raw.githubusercontent.com/MISP/MISP/2.4/VERSION.json') + if r.status_code == 200: + master_version = json.loads(r.text) + return {'version': '{}.{}.{}'.format(master_version['major'], master_version['minor'], master_version['hotfix'])} + return {'error': 'Impossible to retrieve the version of the master branch.'} + + # ## BEGIN Taxonomies ### + + def update_taxonomies(self): + """Update all the taxonomies.""" + response = self._prepare_request('POST', 'taxonomies/update') + return self._check_response(response, expect_json=True) + + def taxonomies(self, pythonify: bool=False): + """Get all the taxonomies.""" + taxonomies = self._prepare_request('GET', 'taxonomies') + taxonomies = self._check_response(taxonomies, expect_json=True) + if not pythonify or 'errors' in taxonomies: + return taxonomies + to_return = [] + for taxonomy in taxonomies: + t = MISPTaxonomy() + t.from_dict(**taxonomy) + to_return.append(t) return to_return - def toggle_warninglist(self, warninglist_id: List[int]=None, warninglist_name: List[str]=None, force_enable: bool=None): + def get_taxonomy(self, taxonomy_id: int, pythonify: bool=False): + """Get a taxonomy by id.""" + taxonomy = self._prepare_request('GET', f'taxonomies/view/{taxonomy_id}') + taxonomy = self._check_response(taxonomy, expect_json=True) + if not pythonify or 'errors' in taxonomy: + return taxonomy + t = MISPTaxonomy() + t.from_dict(**taxonomy) + return t + + def enable_taxonomy(self, taxonomy_id: int): + """Enable a taxonomy by id.""" + response = self._prepare_request('POST', f'taxonomies/enable/{taxonomy_id}') + return self._check_response(response, expect_json=True) + + def disable_taxonomy(self, taxonomy_id: int): + """Disable a taxonomy by id.""" + self.disable_taxonomy_tags(taxonomy_id) + response = self._prepare_request('POST', f'taxonomies/disable/{taxonomy_id}') + return self._check_response(response, expect_json=True) + + def disable_taxonomy_tags(self, taxonomy_id: int): + """Disable all the tags of a taxonomy by id.""" + response = self._prepare_request('POST', 'taxonomies/disableTag/{taxonomy_id}') + return self._check_response(response, expect_json=True) + + def enable_taxonomy_tags(self, taxonomy_id: int): + """Enable all the tags of a taxonomy by id. + NOTE: this automatically done when you call enable_taxonomy.""" + taxonomy = self.get_taxonomy(taxonomy_id) + if not taxonomy['Taxonomy']['enabled']: + raise PyMISPError(f"The taxonomy {taxonomy['Taxonomy']['name']} is not enabled.") + url = urljoin(self.root_url, 'taxonomies/addTag/{}'.format(taxonomy_id)) + response = self._prepare_request('POST', url) + return self._check_response(response, expect_json=True) + + # ## END Taxonomies ### + + # ## BEGIN Tags ### + + def tags(self, pythonify: bool=False): + """Get the list of existing tags.""" + tags = self._prepare_request('GET', 'tags') + tags = self._check_response(tags, expect_json=True) + if not pythonify or 'errors' in tags: + return tags['Tag'] + to_return = [] + for tag in tags['Tag']: + t = MISPTag() + t.from_dict(**tag) + to_return.append(t) + return to_return + + def get_tag(self, tag_id: int, pythonify: bool=False): + """Get a tag by id.""" + tag = self._prepare_request('GET', f'tags/view/{tag_id}') + tag = self._check_response(tag, expect_json=True) + if not pythonify or 'errors' in tag: + return tag + t = MISPTag() + t.from_dict(**tag) + return t + + def add_tag(self, tag: MISPTag, pythonify: bool=True): + '''Add a new tag on a MISP instance''' + new_tag = self._prepare_request('POST', 'tags/add', data=tag) + new_tag = self._check_response(new_tag, expect_json=True) + if not pythonify or 'errors' in new_tag: + return new_tag + t = MISPTag() + t.from_dict(**new_tag) + return t + + def enable_tag(self, tag: MISPTag, pythonify: bool=False): + """Enable a tag.""" + tag.hide_tag = False + return self.update_tag(tag, pythonify=pythonify) + + def disable_tag(self, tag: MISPTag, pythonify: bool=False): + """Disable a tag.""" + tag.hide_tag = True + return self.update_tag(tag, pythonify=pythonify) + + def update_tag(self, tag: MISPTag, tag_id: int=None, pythonify: bool=False): + """Edit only the provided parameters of a tag.""" + if tag_id is None: + if tag.get('id') is None: + raise PyMISPError('The name of the tag you want to update is required. Either directly in the parameters of the method or in the tag itself.') + tag_id = tag.id + # FIXME: inconsistency in MISP: https://github.com/MISP/MISP/issues/4852 + updated_tag = self._prepare_request('POST', f'tags/edit/{tag_id}', {'Tag': tag}) + updated_tag = self._check_response(updated_tag, expect_json=True) + if not pythonify or 'errors' in updated_tag: + return updated_tag + t = MISPTag() + t.from_dict(**updated_tag) + return t + + def delete_tag(self, tag_id: int): + response = self._prepare_request('POST', f'tags/delete/{tag_id}') + return self._check_response(response, expect_json=True) + + # ## END Tags ### + + # ## BEGIN Warninglists ### + + def warninglists(self, pythonify: bool=False): + """Get all the warninglists.""" + warninglists = self._prepare_request('GET', 'warninglists') + warninglists = self._check_response(warninglists, expect_json=True) + if not pythonify or 'errors' in warninglists: + return warninglists['Warninglists'] + to_return = [] + for warninglist in warninglists['Warninglists']: + w = MISPWarninglist() + w.from_dict(**warninglist) + to_return.append(w) + return to_return + + def get_warninglist(self, warninglist_id: int, pythonify: bool=False): + """Get a warninglist by id.""" + warninglist = self._prepare_request('GET', f'warninglists/view/{warninglist_id}') + warninglist = self._check_response(warninglist, expect_json=True) + if not pythonify or 'errors' in warninglist: + return warninglist + w = MISPWarninglist() + w.from_dict(**warninglist) + return w + + def toggle_warninglist(self, warninglist_id: List[int]=None, warninglist_name: List[str]=None, + force_enable: bool=False): '''Toggle (enable/disable) the status of a warninglist by ID. :param warninglist_id: ID of the WarningList :param force_enable: Force the warning list in the enabled state (does nothing is already enabled) ''' - return super().toggle_warninglist(warninglist_id, warninglist_name, force_enable) + if warninglist_id is None and warninglist_name is None: + raise PyMISPError('Either warninglist_id or warninglist_name is required.') + query = {} + if warninglist_id is not None: + if not isinstance(warninglist_id, list): + warninglist_id = [warninglist_id] + query['id'] = warninglist_id + if warninglist_name is not None: + if not isinstance(warninglist_name, list): + warninglist_name = [warninglist_name] + query['name'] = warninglist_name + if force_enable: + query['enabled'] = force_enable + response = self._prepare_request('POST', 'warninglists/toggleEnable', data=json.dumps(query)) + return self._check_response(response, expect_json=True) - def make_timestamp(self, value: DateTypes): - if isinstance(value, datetime): - return datetime.timestamp() - elif isinstance(value, date): - return datetime.combine(value, datetime.max.time()).timestamp() - elif isinstance(value, str): - if value.isdigit(): - return value - else: - try: - float(value) - return value - except ValueError: - # The value can also be '1d', '10h', ... - return value - else: - return value + def update_warninglists(self): + """Update all the warninglists.""" + response = self._prepare_request('POST', 'warninglists/update') + return self._check_response(response, expect_json=True) - def _check_response(self, response, lenient_response_type=False): - """Check if the response from the server is not an unexpected error""" - if response.status_code >= 500: - logger.critical(everything_broken.format(response.request.headers, response.request.body, response.text)) - raise MISPServerError('Error code 500:\n{}'.format(response.text)) - elif 400 <= response.status_code < 500: - # The server returns a json message with the error details - error_message = response.json() - logger.error(f'Something went wrong ({response.status_code}): {error_message}') - return {'errors': (response.status_code, error_message)} + def enable_warninglist(self, warninglist_id: int): + """Enable a warninglist by id.""" + return self.toggle_warninglist(warninglist_id=warninglist_id, force_enable=True) - # At this point, we had no error. + def disable_warninglist(self, warninglist_id: int): + """Disable a warninglist by id.""" + return self.toggle_warninglist(warninglist_id=warninglist_id, force_enable=False) - try: - response = response.json() - if logger.isEnabledFor(logging.DEBUG): - logger.debug(response) - if isinstance(response, dict) and response.get('response') is not None: - # Cleanup. - response = response['response'] - return response - except Exception: - if lenient_response_type and not response.headers.get('content-type').startswith('application/json;'): - return response.text - if logger.isEnabledFor(logging.DEBUG): - logger.debug(response.text) - if not len(response.content): - # Empty response - logger.error('Got an empty response.') - return {'errors': 'The response is empty.'} - return response.text + def values_in_warninglist(self, value: list): + """Check if IOC values are in warninglist""" + response = self._prepare_request('POST', 'warninglists/checkValue', data=json.dumps(value)) + return self._check_response(response, expect_json=True) - def get_event(self, event_id: int): - event = super().get_event(event_id) + # ## END Warninglists ### + + # FIXME: ids can be UUID, str, or integer + # ## BEGIN Event ### + + def get_event(self, event_id: int, pythonify: bool=True): + event = self._prepare_request('GET', f'events/{event_id}') + event = self._check_response(event, expect_json=True) + if not pythonify or 'errors' in event: + return event e = MISPEvent() e.load(event) return e - def add_object(self, event_id: int, misp_object: MISPObject): - created_object = super().add_object(event_id, misp_object) - if isinstance(created_object, str): - raise PyMISPUnexpectedResponse(f'Unexpected response from server: {created_object}') - elif 'errors' in created_object: - return created_object - o = MISPObject(misp_object.name) - o.from_dict(**created_object) - return o - - def update_object(self, misp_object: MISPObject): - updated_object = super().edit_object(misp_object) - if isinstance(updated_object, str): - raise PyMISPUnexpectedResponse(f'Unexpected response from server: {updated_object}') - elif 'errors' in updated_object: - return updated_object - o = MISPObject(misp_object.name) - o.from_dict(**updated_object) - return o - - def get_object(self, object_id: int): - """Get an object - - :param obj_id: Object id to get - """ - misp_object = super().get_object(object_id) - if isinstance(misp_object, str): - raise PyMISPUnexpectedResponse(f'Unexpected response from server: {misp_object}') - elif 'errors' in misp_object: - return misp_object - o = MISPObject(misp_object['Object']['name']) - o.from_dict(**misp_object) - return o - - def add_event(self, event: MISPEvent): - created_event = super().add_event(event) - if isinstance(created_event, str): - raise PyMISPUnexpectedResponse(f'Unexpected response from server: {created_event}') - elif 'errors' in created_event: - return created_event + def add_event(self, event: MISPEvent, pythonify: bool=True): + '''Add a new event on a MISP instance''' + new_event = self._prepare_request('POST', 'events', data=event) + new_event = self._check_response(new_event, expect_json=True) + if not pythonify or 'errors' in new_event: + return new_event e = MISPEvent() - e.load(created_event) + e.load(new_event) return e - def update_event(self, event: MISPEvent): - updated_event = super().update_event(event.uuid, event) - if isinstance(updated_event, str): - raise PyMISPUnexpectedResponse(f'Unexpected response from server: {updated_event}') - elif 'errors' in updated_event: + def update_event(self, event: MISPEvent, event_id: int=None, pythonify: bool=True): + '''Update an event on a MISP instance''' + if event_id is None: + if event.get('id') is None: + raise PyMISPError('The ID of the event you want to update is required. Either directly in the parameters of the method or in the event itself.') + event_id = event.id + updated_event = self._prepare_request('POST', f'events/{event_id}', data=event) + updated_event = self._check_response(updated_event, expect_json=True) + if not pythonify or 'errors' in updated_event: return updated_event e = MISPEvent() e.load(updated_event) return e - def get_attribute(self, attribute_id: int): - attribute = super().get_attribute(attribute_id) + def delete_event(self, event_id: int): + response = self._prepare_request('DELETE', f'events/delete/{event_id}') + return self._check_response(response, expect_json=True) + + def publish(self, event_id: int, alert: bool=False): + """Publish the event with one single HTTP POST. + The default is to not send a mail as it is assumed this method is called on update. + """ + if alert: + response = self._prepare_request('POST', f'events/alert/{event_id}') + else: + response = self._prepare_request('POST', f'events/publish/{event_id}') + return self._check_response(response, expect_json=True) + + # ## END Event ### + + # ## BEGIN Object ### + + def get_object(self, object_id: int, pythonify: bool=True): + misp_object = self._prepare_request('GET', f'objects/view/{object_id}') + misp_object = self._check_response(misp_object, expect_json=True) + if not pythonify or 'errors' in misp_object: + return misp_object + o = MISPObject(misp_object['Object']['name']) + o.from_dict(**misp_object) + return o + + def add_object(self, event_id: int, misp_object: MISPObject, pythonify: bool=True): + '''Add a MISP Object to an existing MISP event''' + new_object = self._prepare_request('POST', f'objects/add/{event_id}', data=misp_object) + new_object = self._check_response(new_object, expect_json=True) + if not pythonify or 'errors' in new_object: + return new_object + o = MISPObject(new_object['Object']['name']) + o.from_dict(**new_object) + return o + + def update_object(self, misp_object: MISPObject, object_id: int=None, pythonify: bool=True): + if object_id is None: + if misp_object.get('id') is None: + raise PyMISPError('The ID of the object you want to update is required. Either directly in the parameters of the method or in the object itself.') + object_id = misp_object.id + updated_object = self._prepare_request('POST', f'objects/edit/{object_id}', data=misp_object) + updated_object = self._check_response(updated_object, expect_json=True) + if not pythonify or 'errors' in updated_object: + return updated_object + o = MISPObject(updated_object['Object']['name']) + o.from_dict(**updated_object) + return o + + def delete_object(self, object_id: int): + # FIXME: MISP doesn't support DELETE on this endpoint + response = self._prepare_request('POST', f'objects/delete/{object_id}') + return self._check_response(response, expect_json=True) + + def add_object_reference(self, misp_object_reference: MISPObjectReference, pythonify: bool=False): + """Add a reference to an object""" + object_reference = self._prepare_request('POST', 'object_references/add', misp_object_reference) + object_reference = self._check_response(object_reference, expect_json=True) + if not pythonify or 'errors' in object_reference: + return object_reference + r = MISPObjectReference() + r.from_dict(**object_reference) + return r + + def delete_object_reference(self, object_reference_id: int): + response = self._prepare_request('POST', f'object_references/delete/{object_reference_id}') + return self._check_response(response, expect_json=True) + + def object_templates(self, pythonify=False): + """Get all the object templates.""" + object_templates = self._prepare_request('GET', 'objectTemplates') + object_templates = self._check_response(object_templates, expect_json=True) + if not pythonify or 'errors' in object_templates: + return object_templates + to_return = [] + for object_template in object_templates: + o = MISPObjectTemplate() + o.from_dict(**object_template) + to_return.append(o) + return to_return + + def get_object_template(self, object_id: int, pythonify=False): + """Gets the full object template corresponting the UUID passed as parameter""" + object_template = self._prepare_request('GET', f'objectTemplates/view/{object_id}') + object_template = self._check_response(object_template, expect_json=True) + if not pythonify or 'errors' in object_template: + return object_template + t = MISPObjectTemplate() + t.from_dict(**object_template) + return t + + def update_object_templates(self): + response = self._prepare_request('POST', 'objectTemplates/update') + return self._check_response(response, expect_json=True) + + # ## END Object ### + + # ## BEGIN Attribute ### + + def get_attribute(self, attribute_id: int, pythonify: bool=True): + attribute = self._prepare_request('GET', f'attributes/view/{attribute_id}') + attribute = self._check_response(attribute, expect_json=True) + if not pythonify or 'errors' in attribute: + return attribute a = MISPAttribute() a.from_dict(**attribute) return a - def add_attribute(self, event_id: int, attribute: MISPAttribute): - url = urljoin(self.root_url, 'attributes/add/{}'.format(event_id)) - response = self._prepare_request('POST', url, data=attribute) - new_attribute = self._check_response(response) - if isinstance(new_attribute, str): - raise PyMISPUnexpectedResponse(f'Unexpected response from server: {new_attribute}') - elif 'errors' in new_attribute: + def add_attribute(self, event_id: int, attribute: MISPAttribute, pythonify: bool=True): + '''Add an attribute to an existing MISP event''' + new_attribute = self._prepare_request('POST', f'attributes/add/{event_id}', data=attribute) + if new_attribute.status_code == 403: + # Re-try with a proposal + return self.add_attribute_proposal(event_id, attribute, pythonify) + new_attribute = self._check_response(new_attribute, expect_json=True) + if not pythonify or 'errors' in new_attribute: return new_attribute a = MISPAttribute() a.from_dict(**new_attribute) return a - def add_attribute_proposal(self, event_id: int, attribute: MISPAttribute): - url = urljoin(self.root_url, 'shadow_attributes/add/{}'.format(event_id)) - response = self._prepare_request('POST', url, attribute) - new_attribute_proposal = self._check_response(response) - if isinstance(new_attribute_proposal, str): - raise PyMISPUnexpectedResponse(f'Unexpected response from server: {new_attribute_proposal}') - elif 'errors' in new_attribute_proposal: - return new_attribute_proposal - a = MISPShadowAttribute() - a.from_dict(**new_attribute_proposal) - return a - - def update_attribute(self, attribute: MISPAttribute): - updated_attribute = super().update_attribute(attribute.uuid, attribute) - if isinstance(updated_attribute, str): - raise PyMISPUnexpectedResponse(f'Unexpected response from server: {updated_attribute}') - elif 'errors' in updated_attribute: + def update_attribute(self, attribute: MISPAttribute, attribute_id: int=None, pythonify: bool=True): + if attribute_id is None: + if attribute.get('uuid'): + attribute_id = attribute.uuid + elif attribute.get('id'): + attribute_id = attribute.id + else: + raise PyMISPError('The ID of the attribute you want to update is required. Either directly in the parameters of the method or in the attribute itself.') + updated_attribute = self._prepare_request('POST', f'attributes/edit/{attribute_id}', data=attribute) + if updated_attribute.status_code == 403: + # Re-try with a proposal + return self.update_attribute_proposal(attribute_id, attribute, pythonify) + updated_attribute = self._check_response(updated_attribute, expect_json=True) + if not pythonify or 'errors' in updated_attribute: return updated_attribute a = MISPAttribute() a.from_dict(**updated_attribute) return a - def update_attribute_proposal(self, attribute_id: int, attribute: MISPAttribute): - url = urljoin(self.root_url, 'shadow_attributes/edit/{}'.format(attribute_id)) - # FIXME: Inconsistency on MISP side - attribute = {'ShadowAttribute': attribute} - response = self._prepare_request('POST', url, attribute) - attribute_proposal = self._check_response(response) - if isinstance(attribute_proposal, str): - raise PyMISPUnexpectedResponse(f'Unexpected response from server: {attribute_proposal}') - elif 'errors' in attribute_proposal: + def delete_attribute(self, attribute_id: int): + response = self._prepare_request('POST', f'attributes/delete/{attribute_id}') + if response.status_code == 403: + # Re-try with a proposal + return self.delete_attribute_proposal(attribute_id) + return self._check_response(response, expect_json=True) + + # ## END Attribute ### + + # ## BEGIN Attribute Proposal ### + + def get_attribute_proposal(self, proposal_id: int, pythonify: bool=True): + attribute_proposal = self._prepare_request('GET', f'shadow_attributes/view/{proposal_id}') + attribute_proposal = self._check_response(attribute_proposal, expect_json=True) + if not pythonify or 'errors' in attribute_proposal: return attribute_proposal a = MISPShadowAttribute() a.from_dict(**attribute_proposal) return a - def get_attribute_proposal(self, proposal_id: int): - url = urljoin(self.root_url, 'shadow_attributes/view/{}'.format(proposal_id)) - response = self._prepare_request('GET', url) - attribute_proposal = self._check_response(response) - if isinstance(attribute_proposal, str): - raise PyMISPUnexpectedResponse(f'Unexpected response from server: {attribute_proposal}') - elif 'errors' in attribute_proposal: - return attribute_proposal + # NOTE: the tree following method have a very specific meaning, look at the comments + + def add_attribute_proposal(self, event_id: int, attribute: MISPAttribute, pythonify: bool=True): + '''Propose a new attribute in an event''' + # FIXME: attribute needs to be a complete MISPAttribute: https://github.com/MISP/MISP/issues/4868 + new_attribute_proposal = self._prepare_request('POST', f'shadow_attributes/add/{event_id}', data=attribute) + new_attribute_proposal = self._check_response(new_attribute_proposal, expect_json=True) + if not pythonify or 'errors' in new_attribute_proposal: + return new_attribute_proposal a = MISPShadowAttribute() - a.from_dict(**attribute_proposal) + a.from_dict(**new_attribute_proposal) return a + def update_attribute_proposal(self, attribute_id: int, attribute: MISPAttribute, pythonify: bool=True): + '''Propose a change for an attribute''' + # FIXME: inconsistency in MISP: https://github.com/MISP/MISP/issues/4857 + attribute = {'ShadowAttribute': attribute} + update_attribute_proposal = self._prepare_request('POST', f'shadow_attributes/edit/{attribute_id}', data=attribute) + update_attribute_proposal = self._check_response(update_attribute_proposal, expect_json=True) + if not pythonify or 'errors' in update_attribute_proposal: + return update_attribute_proposal + a = MISPShadowAttribute() + a.from_dict(**update_attribute_proposal) + return a + + def delete_attribute_proposal(self, attribute_id: int): + '''Propose the deletion of an attribute''' + response = self._prepare_request('POST', f'shadow_attributes/delete/{attribute_id}') + return self._check_response(response, expect_json=True) + + # NOTE: You cannot modify an existing proposal, only accept/discard + def accept_attribute_proposal(self, proposal_id: int): - url = urljoin(self.root_url, 'shadow_attributes/accept/{}'.format(proposal_id)) - response = self._prepare_request('POST', url) - r = self._check_response(response) - if isinstance(r, str): - raise PyMISPUnexpectedResponse(f'Unexpected response from server: {r}') - return r + '''Accept a proposal''' + response = self._prepare_request('POST', f'shadow_attributes/accept/{proposal_id}') + return self._check_response(response, expect_json=True) def discard_attribute_proposal(self, proposal_id: int): - url = urljoin(self.root_url, 'shadow_attributes/discard/{}'.format(proposal_id)) - response = self._prepare_request('POST', url) - r = self._check_response(response) - if isinstance(r, str): - raise PyMISPUnexpectedResponse(f'Unexpected response from server: {r}') - return r + '''Discard a proposal''' + response = self._prepare_request('POST', f'shadow_attributes/discard/{proposal_id}') + return self._check_response(response, expect_json=True) - def add_user(self, user: MISPUser): - user = super().add_user(user) - if isinstance(user, str): - raise PyMISPUnexpectedResponse(f'Unexpected response from server: {user}') - elif 'errors' in user: + # ## END Attribute Proposal ### + + # ## BEGIN Noticelist ### + + def noticelists(self, pythonify=False): + """Get all the noticelists.""" + noticelists = self._prepare_request('GET', 'noticelists') + noticelists = self._check_response(noticelists, expect_json=True) + if not pythonify or 'errors' in noticelists: + return noticelists + to_return = [] + for noticelist in noticelists: + n = MISPNoticelist() + n.from_dict(**noticelist) + to_return.append(n) + return to_return + + def get_noticelist(self, noticelist_id: int, pythonify=False): + """Get a noticelist by id.""" + noticelist = self._prepare_request('GET', f'noticelists/view/{noticelist_id}') + noticelist = self._check_response(noticelist, expect_json=True) + if not pythonify or 'errors' in noticelist: + return noticelist + n = MISPNoticelist() + n.from_dict(**noticelist) + return n + + def enable_noticelist(self, noticelist_id): + """Enable a noticelist by id.""" + # FIXME: https://github.com/MISP/MISP/issues/4856 + # response = self._prepare_request('POST', f'noticelists/enable/{noticelist_id}') + response = self._prepare_request('POST', f'noticelists/enableNoticelist/{noticelist_id}/true') + return self._check_response(response, expect_json=True) + + def disable_noticelist(self, noticelist_id): + """Disable a noticelist by id.""" + # FIXME: https://github.com/MISP/MISP/issues/4856 + # response = self._prepare_request('POST', f'noticelists/disable/{noticelist_id}') + response = self._prepare_request('POST', f'noticelists/enableNoticelist/{noticelist_id}') + return self._check_response(response, expect_json=True) + + def update_noticelists(self): + """Update all the noticelists.""" + response = self._prepare_request('POST', 'noticelists/update') + return self._check_response(response, expect_json=True) + + # ## END Galaxy ### + + # ## BEGIN Galaxy ### + + def galaxies(self, pythonify=False): + """Get all the galaxies.""" + galaxies = self._prepare_request('GET', 'galaxies') + galaxies = self._check_response(galaxies, expect_json=True) + if not pythonify or 'errors' in galaxies: + return galaxies + to_return = [] + for galaxy in galaxies: + g = MISPGalaxy() + g.from_dict(**galaxy) + to_return.append(g) + return to_return + + def get_galaxy(self, galaxy_id: int, pythonify=False): + """Get a galaxy by id.""" + galaxy = self._prepare_request('GET', f'galaxies/view/{galaxy_id}') + galaxy = self._check_response(galaxy, expect_json=True) + if not pythonify or 'errors' in galaxy: + return galaxy + g = MISPGalaxy() + g.from_dict(**galaxy) + return g + + def update_galaxies(self): + """Update all the galaxies.""" + response = self._prepare_request('POST', 'galaxies/update') + return self._check_response(response, expect_json=True) + + # ## END Galaxy ### + + # ## BEGIN User ### + + def users(self, pythonify=False): + """Get all the users.""" + users = self._prepare_request('GET', 'admin/users') + users = self._check_response(users, expect_json=True) + if not pythonify or 'errors' in users: + return users + to_return = [] + for user in users: + u = MISPUser() + u.from_dict(**user) + to_return.append(u) + return to_return + + def get_user(self, user_id: int='me', pythonify: bool=False): + '''Get a user. `me` means the owner of the API key doing the query.''' + user = self._prepare_request('GET', f'users/view/{user_id}') + user = self._check_response(user, expect_json=True) + if not pythonify or 'errors' in user: return user u = MISPUser() u.from_dict(**user) return u - def get_user(self, userid='me'): - user = super().get_user(userid) - if isinstance(user, str): - raise PyMISPUnexpectedResponse(f'Unexpected response from server: {user}') - elif 'errors' in user: + def add_user(self, user: MISPUser, pythonify: bool=False): + user = self._prepare_request('POST', f'admin/users/add', data=user) + user = self._check_response(user, expect_json=True) + if not pythonify or 'errors' in user: return user u = MISPUser() u.from_dict(**user) return u - def add_organisation(self, organisation: MISPOrganisation): - organisation = super().add_organisation(organisation) - if isinstance(organisation, str): - raise PyMISPUnexpectedResponse(f'Unexpected response from server: {organisation}') - elif 'errors' in organisation: + def update_user(self, user: MISPUser, user_id: int=None, pythonify: bool=False): + '''Update an event on a MISP instance''' + if user_id is None: + if user.get('id') is None: + raise PyMISPError('The ID of the user you want to update is required. Either directly in the parameters of the method or in the user itself.') + user_id = user.id + updated_user = self._prepare_request('POST', f'admin/users/edit/{user_id}', data=user) + updated_user = self._check_response(updated_user, expect_json=True) + if not pythonify or 'errors' in updated_user: + return updated_user + e = MISPUser() + e.from_dict(**updated_user) + return e + + def delete_user(self, user_id: int): + # NOTE: MISP in inconsistent and currently require "delete" in the path and doesn't support HTTP DELETE + response = self._prepare_request('POST', f'admin/users/delete/{user_id}') + return self._check_response(response, expect_json=True) + + # ## END User ### + + # ## BEGIN Organisation ### + + def organisations(self, scope="local", pythonify=False): + """Get all the organisations.""" + organisations = self._prepare_request('GET', f'organisations/index/scope:{scope}') + organisations = self._check_response(organisations, expect_json=True) + if not pythonify or 'errors' in organisations: + return organisations + to_return = [] + for organisation in organisations: + o = MISPOrganisation() + o.from_dict(**organisation) + to_return.append(o) + return to_return + + def get_organisation(self, organisation_id: int, pythonify: bool=True): + '''Get an organisation.''' + organisation = self._prepare_request('GET', f'organisations/view/{organisation_id}') + organisation = self._check_response(organisation, expect_json=True) + if not pythonify or 'errors' in organisation: return organisation o = MISPOrganisation() o.from_dict(**organisation) return o + def add_organisation(self, organisation: MISPOrganisation, pythonify: bool=True): + new_organisation = self._prepare_request('POST', f'admin/organisations/add', data=organisation) + new_organisation = self._check_response(new_organisation, expect_json=True) + if not pythonify or 'errors' in new_organisation: + return new_organisation + o = MISPOrganisation() + o.from_dict(**new_organisation) + return o + + def update_organisation(self, organisation: MISPOrganisation, organisation_id: int=None, pythonify: bool=True): + '''Update an organisation''' + if organisation_id is None: + if organisation.get('id') is None: + raise PyMISPError('The ID of the organisation you want to update is required. Either directly in the parameters of the method or in the organisation itself.') + organisation_id = organisation.id + updated_organisation = self._prepare_request('POST', f'admin/organisations/edit/{organisation_id}', data=organisation) + updated_organisation = self._check_response(updated_organisation, expect_json=True) + if not pythonify or 'errors' in updated_organisation: + return updated_organisation + o = MISPOrganisation() + o.from_dict(**organisation) + return o + + def delete_organisation(self, organisation_id: int): + # NOTE: MISP in inconsistent and currently require "delete" in the path and doesn't support HTTP DELETE + response = self._prepare_request('POST', f'admin/organisations/delete/{organisation_id}') + return self._check_response(response, expect_json=True) + + # ## END Organisation ### + + # ## BEGIN Sighting ### + + def sightings(self, misp_entity: AbstractMISP, org_id: int=None, pythonify=False): + """Get the list of sighting related to a MISPEvent or a MISPAttribute (depending on type of misp_entity)""" + # FIXME: https://github.com/MISP/MISP/issues/4875 + if isinstance(misp_entity, MISPEvent): + scope = 'event' + elif isinstance(misp_entity, MISPAttribute): + scope = 'attribute' + else: + raise PyMISPError('misp_entity can only be a MISPEvent or a MISPAttribute') + if org_id is not None: + url = f'sightings/listSightings/{misp_entity.id}/{scope}/{org_id}' + else: + url = f'sightings/listSightings/{misp_entity.id}/{scope}' + sightings = self._prepare_request('POST', url) + sightings = self._check_response(sightings, expect_json=True) + if not pythonify or 'errors' in sightings: + return sightings + to_return = [] + for sighting in sightings: + s = MISPSighting() + s.from_dict(**sighting) + to_return.append(s) + return to_return + + def add_sighting(self, sighting: MISPSighting, attribute_id: int=None): + # FIXME: no pythonify possible: https://github.com/MISP/MISP/issues/4867 + pythonify = False + if attribute_id: + new_sighting = self._prepare_request('POST', f'sightings/add/{attribute_id}', data=sighting) + else: + # Either the ID/UUID is in the sighting, or we want to add a sighting on all the attributes with the given value + new_sighting = self._prepare_request('POST', f'sightings/add', data=sighting) + new_sighting = self._check_response(new_sighting, expect_json=True) + if not pythonify or 'errors' in new_sighting: + return new_sighting + s = MISPSighting() + s.from_dict(**new_sighting) + return s + + # ## END Sighting ### + + # ## BEGIN Statistics ### + + def attributes_statistics(self, context: str='type', percentage: bool=False): + """Get attributes statistics from the MISP instance.""" + # FIXME: https://github.com/MISP/MISP/issues/4874 + if context not in ['type', 'category']: + raise PyMISPError('context can only be "type" or "category"') + if percentage: + path = f'attributes/attributeStatistics/{context}/true' + else: + path = f'attributes/attributeStatistics/{context}' + response = self._prepare_request('GET', path) + return self._check_response(response, expect_json=True) + + def tags_statistics(self, percentage: bool=False, name_sort: bool=False): + """Get tags statistics from the MISP instance""" + # FIXME: https://github.com/MISP/MISP/issues/4874 + # NOTE: https://github.com/MISP/MISP/issues/4879 + if percentage: + percentage = 'true' + else: + percentage = 'false' + if name_sort: + name_sort = 'true' + else: + name_sort = 'false' + response = self._prepare_request('GET', f'tags/tagStatistics/{percentage}/{name_sort}') + return self._check_response(response) + + def users_statistics(self, context: str='data'): + """Get users statistics from the MISP instance""" + # FIXME: https://github.com/MISP/MISP/issues/4874 + availables_contexts = ['data', 'orgs', 'users', 'tags', 'attributehistogram', 'sightings', 'galaxyMatrix'] + if context not in availables_contexts: + raise PyMISPError("context can only be {','.join(availables_contexts)}") + response = self._prepare_request('GET', f'users/statistics/{context}.json') + return self._check_response(response) + + # ## END Statistics ### + + # ## BEGIN Others ### + + def push_event_to_ZMQ(self, event_id: int): + """Force push an event on ZMQ""" + response = self._prepare_request('POST', f'events/pushEventToZMQ/{event_id}.json') + return self._check_response(response, expect_json=True) + + def direct_call(self, url: str, data: dict=None, params: dict={}): + '''Very lightweight call that posts a data blob (python dictionary or json string) on the URL''' + if data is None: + response = self._prepare_request('GET', url, params=params) + else: + response = self._prepare_request('POST', url, data=data, params=params) + return self._check_response(response, lenient_response_type=True) + + def freetext(self, event_id: int, string: str, adhereToWarninglists: Union[bool, str]=False, + distribution: int=None, returnMetaAttributes: bool=False, pythonify=False): + """Pass a text to the freetext importer""" + query = {"value": string} + wl_params = [False, True, 'soft'] + if adhereToWarninglists in wl_params: + query['adhereToWarninglists'] = adhereToWarninglists + else: + raise Exception('Invalid parameter, adhereToWarninglists Can only be {}'.format(', '.join(wl_params))) + if distribution is not None: + query['distribution'] = distribution + if returnMetaAttributes: + query['returnMetaAttributes'] = returnMetaAttributes + attributes = self._prepare_request('POST', f'events/freeTextImport/{event_id}', data=query) + attributes = self._check_response(attributes, expect_json=True) + if returnMetaAttributes or not pythonify or 'errors' in attributes: + return attributes + to_return = [] + for attribute in attributes: + a = MISPAttribute() + a.from_dict(**attribute) + to_return.append(a) + return to_return + + # ## END Others ### + + # ## BEGIN Sharing group ### + + def sharing_groups(self, pythonify: bool=False): + """Get the existing sharing groups""" + sharing_groups = self._prepare_request('GET', 'sharing_groups') + sharing_groups = self._check_response(sharing_groups, expect_json=True) + if not pythonify or 'errors' in sharing_groups: + return sharing_groups + to_return = [] + for sharing_group in sharing_groups: + s = MISPSharingGroup() + s.from_dict(**sharing_group) + to_return.append(s) + return to_return + + def add_sharing_group(self, sharing_group: MISPSharingGroup, pythonify: bool=True): + sharing_group = self._prepare_request('POST', f'sharing_groups/add', data=sharing_group) + sharing_group = self._check_response(sharing_group, expect_json=True) + # FIXME: https://github.com/MISP/MISP/issues/4882 + sharing_group = sharing_group[0] + if not pythonify or 'errors' in sharing_group: + return sharing_group + s = MISPSharingGroup() + s.from_dict(**sharing_group) + return s + + def delete_sharing_group(self, sharing_group_id: int): + response = self._prepare_request('POST', f'sharing_groups/delete/{sharing_group_id}') + return self._check_response(response, expect_json=True) + + def add_org_to_sharing_group(self, sharing_group_id: int, organisation_id: int, extend: bool=False): + '''Add an organisation to a sharing group. + :sharing_group: Sharing group's local instance ID, or Sharing group's global UUID + :organisation: Organisation's local instance ID, or Organisation's global UUID, or Organisation's name as known to the curent instance + :extend: Allow the organisation to extend the group + ''' + to_jsonify = {'sg_id': sharing_group_id, 'org_id': organisation_id, 'extend': extend} + response = self._prepare_request('POST', 'sharingGroups/addOrg', data=to_jsonify) + return self._check_response(response) + + def remove_org_from_sharing_group(self, sharing_group_id: int, organisation_id: int): + '''Remove an organisation from a sharing group. + :sharing_group: Sharing group's local instance ID, or Sharing group's global UUID + :organisation: Organisation's local instance ID, or Organisation's global UUID, or Organisation's name as known to the curent instance + ''' + to_jsonify = {'sg_id': sharing_group_id, 'org_id': organisation_id} + response = self._prepare_request('POST', 'sharingGroups/removeOrg', data=to_jsonify) + return self._check_response(response) + + def add_server_to_sharing_group(self, sharing_group_id: int, server_id: int, all_orgs: bool=False): + '''Add a server to a sharing group. + :sharing_group: Sharing group's local instance ID, or Sharing group's global UUID + :server: Server's local instance ID, or URL of the Server, or Server's name as known to the curent instance + :all_orgs: Add all the organisations of the server to the group + ''' + to_jsonify = {'sg_id': sharing_group_id, 'server_id': server_id, 'all_orgs': all_orgs} + response = self._prepare_request('POST', 'sharingGroups/addServer', data=to_jsonify) + return self._check_response(response) + + def remove_server_from_sharing_group(self, sharing_group_id: int, server_id: int): + '''Remove a server from a sharing group. + :sharing_group: Sharing group's local instance ID, or Sharing group's global UUID + :server: Server's local instance ID, or URL of the Server, or Server's name as known to the curent instance + ''' + to_jsonify = {'sg_id': sharing_group_id, 'server_id': server_id} + response = self._prepare_request('POST', 'sharingGroups/removeServer', data=to_jsonify) + return self._check_response(response) + + # ## END Sharing groups ### + + # ## BEGIN Feed ### + + def feeds(self, pythonify: bool=False): + """Get the list of existing feeds.""" + feeds = self._prepare_request('GET', 'feeds') + feeds = self._check_response(feeds, expect_json=True) + if not pythonify or 'errors' in feeds: + return feeds + to_return = [] + for feed in feeds: + f = MISPFeed() + f.from_dict(**feed) + to_return.append(f) + return to_return + + def get_feed(self, feed_id: int, pythonify: bool=False): + """Get a feed by id.""" + feed = self._prepare_request('GET', f'feeds/view/{feed_id}') + feed = self._check_response(feed, expect_json=True) + if not pythonify or 'errors' in feed: + return feed + f = MISPFeed() + f.from_dict(**feed) + return f + + def add_feed(self, feed: MISPFeed, pythonify: bool=False): + '''Add a new feed on a MISP instance''' + # FIXME: https://github.com/MISP/MISP/issues/4834 + feed = {'Feed': feed} + new_feed = self._prepare_request('POST', 'feeds/add', data=feed) + new_feed = self._check_response(new_feed, expect_json=True) + if not pythonify or 'errors' in new_feed: + return new_feed + f = MISPFeed() + f.from_dict(**new_feed) + return f + + def enable_feed(self, feed_id: int, pythonify: bool=False): + feed = MISPFeed() + feed.id = feed_id + feed.enabled = True + return self.update_feed(feed=feed, pythonify=pythonify) + + def disable_feed(self, feed_id: int, pythonify: bool=False): + feed = MISPFeed() + feed.id = feed_id + feed.enabled = False + return self.update_feed(feed=feed, pythonify=pythonify) + + def enable_feed_cache(self, feed_id: int, pythonify: bool=False): + feed = MISPFeed() + feed.id = feed_id + feed.caching_enabled = True + return self.update_feed(feed=feed, pythonify=pythonify) + + def disable_feed_cache(self, feed_id: int, pythonify: bool=False): + feed = MISPFeed() + feed.id = feed_id + feed.caching_enabled = False + return self.update_feed(feed=feed, pythonify=pythonify) + + def update_feed(self, feed: MISPFeed, feed_id: int=None, pythonify: bool=False): + '''Update a feed on a MISP instance''' + if feed_id is None: + if feed.get('id') is None: + raise PyMISPError('The ID of the feed you want to update is required. Either directly in the parameters of the method or in the user itself.') + feed_id = feed.id + # FIXME: https://github.com/MISP/MISP/issues/4834 + feed = {'Feed': feed} + updated_feed = self._prepare_request('POST', f'feeds/edit/{feed_id}', data=feed) + updated_feed = self._check_response(updated_feed, expect_json=True) + if not pythonify or 'errors' in updated_feed: + return updated_feed + f = MISPFeed() + f.from_dict(**updated_feed) + return f + + def delete_feed(self, feed_id: int): + response = self._prepare_request('POST', f'feeds/delete/{feed_id}') + return self._check_response(response, expect_json=True) + + def fetch_feed(self, feed_id): + """Fetch one single feed""" + response = self._prepare_request('GET', f'feeds/fetchFromFeed/{feed_id}') + return self._check_response(response) + + def cache_all_feeds(self): + """ Cache all the feeds""" + response = self._prepare_request('GET', 'feeds/cacheFeeds/all') + return self._check_response(response) + + def cache_feed(self, feed_id): + """Cache a specific feed""" + response = self._prepare_request('GET', f'feeds/cacheFeeds/{feed_id}') + return self._check_response(response) + + def cache_freetext_feeds(self): + """Cache all the freetext feeds""" + response = self._prepare_request('GET', 'feeds/cacheFeeds/freetext') + return self._check_response(response) + + def cache_misp_feeds(self): + """Cache all the MISP feeds""" + response = self._prepare_request('GET', 'feeds/cacheFeeds/misp') + return self._check_response(response) + + def compare_feeds(self): + """Generate the comparison matrix for all the MISP feeds""" + response = self._prepare_request('GET', 'feeds/compareFeeds') + return self._check_response(response) + + # ## END Feed ### + + # ## BEGIN Role ### + + def roles(self, pythonify: bool=False): + """Get the existing roles""" + roles = self._prepare_request('GET', 'roles') + roles = self._check_response(roles, expect_json=True) + if not pythonify or 'errors' in roles: + return roles + to_return = [] + for role in roles: + r = MISPRole() + r.from_dict(**role) + to_return.append(r) + return to_return + + # ## END Role ### + + # ## BEGIN Server ### + + def servers(self, pythonify=False): + """Get the existing servers""" + servers = self._prepare_request('GET', 'servers') + servers = self._check_response(servers, expect_json=True) + if not pythonify or 'errors' in servers: + return servers + to_return = [] + for server in servers: + s = MISPServer() + s.from_dict(**server) + to_return.append(s) + return to_return + + def add_server(self, server: MISPServer, pythonify: bool=True): + server = self._prepare_request('POST', f'servers/add', data=server) + server = self._check_response(server, expect_json=True) + if not pythonify or 'errors' in server: + return server + s = MISPServer() + s.from_dict(**server) + return s + + def update_server(self, server: MISPServer, server_id: int=None, pythonify: bool=True): + '''Update a server on a MISP instance''' + if server_id is None: + if server.get('id') is None: + raise PyMISPError('The ID of the server you want to update is required. Either directly in the parameters of the method or in the user itself.') + server_id = server.id + updated_server = self._prepare_request('POST', f'servers/edit/{server_id}', data=server) + updated_server = self._check_response(updated_server, expect_json=True) + if not pythonify or 'errors' in updated_server: + return updated_server + s = MISPServer() + s.from_dict(**updated_server) + return s + + def delete_server(self, server_id: int): + response = self._prepare_request('POST', f'servers/delete/{server_id}') + return self._check_response(response, expect_json=True) + + def server_pull(self, server_id: int, event_id: int=None): + # FIXME: POST & data + if event_id: + url = f'servers/pull/{server_id}/{event_id}' + else: + url = f'servers/pull/{server_id}' + response = self._prepare_request('GET', url) + # FIXME: can we pythonify? + return self._check_response(response) + + def server_push(self, server_id: int, event_id: int=None): + # FIXME: POST & data + if event_id: + url = f'servers/push/{server_id}/{event_id}' + else: + url = f'servers/push/{server_id}' + response = self._prepare_request('GET', url) + # FIXME: can we pythonify? + return self._check_response(response) + + # ## END Server ### + + # ## BEGIN Global helpers ### + + def change_sharing_group_on_entity(self, misp_entity: AbstractMISP, sharing_group_id): + """Change the sharing group of an event, an attribute, or an object""" + misp_entity.distribution = 4 # Needs to be 'Sharing group' + if 'SharingGroup' in misp_entity: # Delete former SharingGroup information + del misp_entity.SharingGroup + misp_entity.sharing_group_id = sharing_group_id # Set new sharing group id + if isinstance(misp_entity, MISPEvent): + return self.update_event(misp_entity) + elif isinstance(misp_entity, MISPObject): + return self.update_object(misp_entity) + elif isinstance(misp_entity, MISPAttribute): + return self.update_attribute(misp_entity) + else: + raise PyMISPError('The misp_entity must be MISPEvent, MISPObject or MISPAttribute') + + def tag(self, misp_entity: Union[AbstractMISP, str], tag: str): + """Tag an event or an attribute. misp_entity can be a UUID""" + if 'uuid' in misp_entity: + uuid = misp_entity.uuid + else: + uuid = misp_entity + to_post = {'uuid': uuid, 'tag': tag} + response = self._prepare_request('POST', 'tags/attachTagToObject', data=to_post) + return self._check_response(response, expect_json=True) + + def untag(self, misp_entity: Union[AbstractMISP, str], tag: str): + """Untag an event or an attribute. misp_entity can be a UUID""" + if 'uuid' in misp_entity: + uuid = misp_entity.uuid + else: + uuid = misp_entity + to_post = {'uuid': uuid, 'tag': tag} + response = self._prepare_request('POST', 'tags/removeTagFromObject', data=to_post) + return self._check_response(response, expect_json=True) + + # ## END Global helpers ### + + # ## BEGIN Search methods ### + def search_sightings(self, context: Optional[str]=None, context_id: Optional[SearchType]=None, type_sighting: Optional[str]=None, @@ -322,9 +1248,8 @@ class ExpandedPyMISP(PyMISP): url = urljoin(self.root_url, url_path) response = self._prepare_request('POST', url, data=query) - normalized_response = self._check_response(response) - if isinstance(normalized_response, str) or (isinstance(normalized_response, dict) - and normalized_response.get('errors')): + normalized_response = self._check_response(response, expect_json=True) + if not pythonify or 'errors' in normalized_response: return normalized_response elif pythonify: to_return = [] @@ -495,11 +1420,14 @@ class ExpandedPyMISP(PyMISP): query['headerless'] = headerless url = urljoin(self.root_url, f'{controller}/restSearch') response = self._prepare_request('POST', url, data=query) - normalized_response = self._check_response(response) + if return_format == 'json': + normalized_response = self._check_response(response, expect_json=True) + else: + normalized_response = self._check_response(response) + if return_format == 'csv' and pythonify and not headerless: return self._csv_to_dict(normalized_response) - elif isinstance(normalized_response, str) or (isinstance(normalized_response, dict) - and normalized_response.get('errors')): + elif 'errors' in normalized_response: return normalized_response elif return_format == 'json' and pythonify: # The response is in json, we can convert it to a list of pythonic MISP objects @@ -520,16 +1448,6 @@ class ExpandedPyMISP(PyMISP): else: return normalized_response - def _csv_to_dict(self, csv_content): - '''Makes a list of dict out of a csv file (requires headers)''' - fieldnames, lines = csv_content.split('\n', 1) - fieldnames = fieldnames.split(',') - to_return = [] - for line in csv.reader(lines.split('\n')): - if line: - to_return.append({fname: value for fname, value in zip(fieldnames, line)}) - return to_return - def search_logs(self, limit: Optional[int]=None, page: Optional[int]=None, log_id: Optional[int]=None, title: Optional[str]=None, created: Optional[DateTypes]=None, model: Optional[str]=None, @@ -562,16 +1480,15 @@ class ExpandedPyMISP(PyMISP): if log_id is not None: query['id'] = query.pop('log_id') - url = urljoin(self.root_url, 'admin/logs/index') - response = self._prepare_request('POST', url, data=query) - normalized_response = self._check_response(response) - if not pythonify: + response = self._prepare_request('POST', 'admin/logs/index', data=query) + normalized_response = self._check_response(response, expect_json=True) + if not pythonify or 'errors' in normalized_response: return normalized_response to_return = [] for l in normalized_response: ml = MISPLog() - ml.from_dict(**l['Log']) + ml.from_dict(**l) to_return.append(ml) return to_return @@ -618,7 +1535,7 @@ class ExpandedPyMISP(PyMISP): url = urljoin(self.root_url, 'events/index') response = self._prepare_request('POST', url, data=query) - normalized_response = self._check_response(response) + normalized_response = self._check_response(response, expect_json=True) if not pythonify: return normalized_response @@ -632,7 +1549,7 @@ class ExpandedPyMISP(PyMISP): def set_default_role(self, role_id: int): url = urljoin(self.root_url, f'/admin/roles/set_default/{role_id}') response = self._prepare_request('POST', url) - return self._check_response(response) + return self._check_response(response, expect_json=True) def upload_stix(self, path, version: str='2'): """Upload a STIX file to MISP. @@ -656,3 +1573,119 @@ class ExpandedPyMISP(PyMISP): response = self._prepare_request('POST', url, data=to_post) return response + + # ## END Search methods ### + + # ## Helpers ### + + def make_timestamp(self, value: DateTypes): + '''Catch-all method to normalize anything that can be converted to a timestamp''' + if isinstance(value, datetime): + return datetime.timestamp() + elif isinstance(value, date): + return datetime.combine(value, datetime.max.time()).timestamp() + elif isinstance(value, str): + if value.isdigit(): + return value + else: + try: + float(value) + return value + except ValueError: + # The value can also be '1d', '10h', ... + return value + else: + return value + + def build_complex_query(self, or_parameters: Optional[List[SearchType]]=None, + and_parameters: Optional[List[SearchType]]=None, + not_parameters: Optional[List[SearchType]]=None): + '''Build a complex search query. MISP expects a dictionary with AND, OR and NOT keys.''' + to_return = {} + if and_parameters: + to_return['AND'] = and_parameters + if not_parameters: + to_return['NOT'] = not_parameters + if or_parameters: + to_return['OR'] = or_parameters + return to_return + + # ## Internal methods ### + + def _check_response(self, response, lenient_response_type=False, expect_json=False): + """Check if the response from the server is not an unexpected error""" + if response.status_code >= 500: + logger.critical(everything_broken.format(response.request.headers, response.request.body, response.text)) + raise MISPServerError(f'Error code 500:\n{response.text}') + elif 400 <= response.status_code < 500: + # The server returns a json message with the error details + error_message = response.json() + logger.error(f'Something went wrong ({response.status_code}): {error_message}') + return {'errors': (response.status_code, error_message)} + + # At this point, we had no error. + + try: + response = response.json() + if logger.isEnabledFor(logging.DEBUG): + logger.debug(response) + if isinstance(response, dict) and response.get('response') is not None: + # Cleanup. + response = response['response'] + return response + except Exception: + if logger.isEnabledFor(logging.DEBUG): + logger.debug(response.text) + if expect_json: + raise PyMISPUnexpectedResponse(f'Unexpected response from server: {response.text}') + if lenient_response_type and not response.headers.get('content-type').startswith('application/json'): + return response.text + if not len(response.content): + # Empty response + logger.error('Got an empty response.') + return {'errors': 'The response is empty.'} + return response.text + + def __repr__(self): + return f'<{self.__class__.__name__}(url={self.root_url})' + + def _prepare_request(self, request_type: str, url: str, data: dict={}, params: dict={}, output_type: str='json'): + '''Prepare a request for python-requests''' + url = urljoin(self.root_url, url) + if logger.isEnabledFor(logging.DEBUG): + logger.debug(f'{request_type} - {url}') + if data is not None: + logger.debug(data) + if data: + if not isinstance(data, str): # Else, we already have a text blob to send + if isinstance(data, dict): # Else, we can directly json encode. + # Remove None values. + data = {k: v for k, v in data.items() if v is not None} + data = json.dumps(data, cls=MISPEncode) + + req = requests.Request(request_type, url, data=data, params=params) + with requests.Session() as s: + user_agent = 'PyMISP {__version__} - Python {".".join(str(x) for x in sys.version_info[:2])}' + if self.tool: + user_agent = f'{user_agent} - {self.tool}' + req.auth = self.auth + prepped = s.prepare_request(req) + prepped.headers.update( + {'Authorization': self.key, + 'Accept': f'application/{output_type}', + 'content-type': f'application/{output_type}', + 'User-Agent': user_agent}) + if logger.isEnabledFor(logging.DEBUG): + logger.debug(prepped.headers) + settings = s.merge_environment_settings(req.url, proxies=self.proxies or {}, stream=None, verify=self.ssl, cert=self.cert) + return s.send(prepped, **settings) + + def _csv_to_dict(self, csv_content): + '''Makes a list of dict out of a csv file (requires headers)''' + fieldnames, lines = csv_content.split('\n', 1) + fieldnames = fieldnames.split(',') + to_return = [] + for line in csv.reader(lines.split('\n')): + if line: + to_return.append({fname: value for fname, value in zip(fieldnames, line)}) + return to_return diff --git a/pymisp/data/describeTypes.json b/pymisp/data/describeTypes.json index 71f4fea..6a678f0 100644 --- a/pymisp/data/describeTypes.json +++ b/pymisp/data/describeTypes.json @@ -125,6 +125,10 @@ "default_category": "Network activity", "to_ids": 1 }, + "community-id": { + "default_category": "Network activity", + "to_ids": 1 + }, "pattern-in-file": { "default_category": "Payload installation", "to_ids": 1 @@ -666,6 +670,7 @@ "snort", "bro", "zeek", + "community-id", "pattern-in-file", "pattern-in-traffic", "pattern-in-memory", @@ -1075,7 +1080,8 @@ "hostname|port", "bro", "zeek", - "anonymised" + "anonymised", + "community-id" ], "Payload type": [ "comment", @@ -1145,7 +1151,8 @@ "github-repository", "other", "cortex", - "anonymised" + "anonymised", + "community-id" ], "Financial fraud": [ "btc", diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 3ec8879..33a80df 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -11,17 +11,15 @@ import sys import uuid from collections import defaultdict -from . import deprecated +from deprecated import deprecated + from .abstract import AbstractMISP from .exceptions import UnknownMISPObjectTemplate, InvalidMISPObject, PyMISPError, NewEventError, NewAttributeError import logging logger = logging.getLogger('pymisp') - if sys.version_info < (3, 0): - logger.warning("You're using python 2, it is strongly recommended to use python >=3.6") - # This is required because Python 2 is a pain. from datetime import tzinfo, timedelta @@ -354,23 +352,23 @@ class MISPAttribute(AbstractMISP): signed, _ = c.sign(to_sign, mode=mode.DETACH) self.sig = base64.b64encode(signed).decode() - @deprecated + @deprecated(reason="Use self.known_types instead. Removal date: 2020-01-01.") def get_known_types(self): # pragma: no cover return self.known_types - @deprecated + @deprecated(reason="Use self.malware_binary instead. Removal date: 2020-01-01.") def get_malware_binary(self): # pragma: no cover return self.malware_binary - @deprecated + @deprecated(reason="Use self.to_dict() instead. Removal date: 2020-01-01.") def _json(self): # pragma: no cover return self.to_dict() - @deprecated + @deprecated(reason="Use self.to_dict() instead. Removal date: 2020-01-01.") def _json_full(self): # pragma: no cover return self.to_dict() - @deprecated + @deprecated(reason="Use self.from_dict(**kwargs) instead. Removal date: 2020-01-01.") def set_all_values(self, **kwargs): # pragma: no cover self.from_dict(**kwargs) @@ -782,15 +780,15 @@ class MISPEvent(AbstractMISP): to_return['global'] = False return to_return - @deprecated + @deprecated(reason="Use self.known_types instead. Removal date: 2020-01-01.") def get_known_types(self): # pragma: no cover return self.known_types - @deprecated + @deprecated(reason="Use self.from_dict(**kwargs) instead. Removal date: 2020-01-01.") def set_all_values(self, **kwargs): # pragma: no cover self.from_dict(**kwargs) - @deprecated + @deprecated(reason="Use self.to_dict() instead. Removal date: 2020-01-01.") def _json(self): # pragma: no cover return self.to_dict() @@ -800,19 +798,28 @@ class MISPObjectReference(AbstractMISP): def __init__(self): super(MISPObjectReference, self).__init__() - def from_dict(self, object_uuid, referenced_uuid, relationship_type, comment=None, **kwargs): - self.object_uuid = object_uuid - self.referenced_uuid = referenced_uuid - self.relationship_type = relationship_type - self.comment = comment + def from_dict(self, **kwargs): + if kwargs.get('ObjectReference'): + kwargs = kwargs.get('ObjectReference') super(MISPObjectReference, self).from_dict(**kwargs) def __repr__(self): - if hasattr(self, 'referenced_uuid'): + if hasattr(self, 'referenced_uuid') and hasattr(self, 'object_uuid'): return '<{self.__class__.__name__}(object_uuid={self.object_uuid}, referenced_uuid={self.referenced_uuid}, relationship_type={self.relationship_type})'.format(self=self) return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) +class MISPObjectTemplate(AbstractMISP): + + def __init__(self): + super(MISPObjectTemplate, self).__init__() + + def from_dict(self, **kwargs): + if kwargs.get('ObjectTemplate'): + kwargs = kwargs.get('ObjectTemplate') + super(MISPObjectTemplate, self).from_dict(**kwargs) + + class MISPUser(AbstractMISP): def __init__(self): @@ -840,12 +847,99 @@ class MISPFeed(AbstractMISP): def __init__(self): super(MISPFeed, self).__init__() + def from_dict(self, **kwargs): + if kwargs.get('Feed'): + kwargs = kwargs.get('Feed') + super(MISPFeed, self).from_dict(**kwargs) + + +class MISPWarninglist(AbstractMISP): + + def __init__(self): + super(MISPWarninglist, self).__init__() + + def from_dict(self, **kwargs): + if kwargs.get('Warninglist'): + kwargs = kwargs.get('Warninglist') + super(MISPWarninglist, self).from_dict(**kwargs) + + +class MISPTaxonomy(AbstractMISP): + + def __init__(self): + super(MISPTaxonomy, self).__init__() + + def from_dict(self, **kwargs): + if kwargs.get('Taxonomy'): + kwargs = kwargs.get('Taxonomy') + super(MISPTaxonomy, self).from_dict(**kwargs) + + +class MISPGalaxy(AbstractMISP): + + def __init__(self): + super(MISPGalaxy, self).__init__() + + def from_dict(self, **kwargs): + if kwargs.get('Galaxy'): + kwargs = kwargs.get('Galaxy') + super(MISPGalaxy, self).from_dict(**kwargs) + + +class MISPNoticelist(AbstractMISP): + + def __init__(self): + super(MISPNoticelist, self).__init__() + + def from_dict(self, **kwargs): + if kwargs.get('Noticelist'): + kwargs = kwargs.get('Noticelist') + super(MISPNoticelist, self).from_dict(**kwargs) + + +class MISPRole(AbstractMISP): + + def __init__(self): + super(MISPRole, self).__init__() + + def from_dict(self, **kwargs): + if kwargs.get('Role'): + kwargs = kwargs.get('Role') + super(MISPRole, self).from_dict(**kwargs) + + +class MISPServer(AbstractMISP): + + def __init__(self): + super(MISPServer, self).__init__() + + def from_dict(self, **kwargs): + if kwargs.get('Server'): + kwargs = kwargs.get('Server') + super(MISPServer, self).from_dict(**kwargs) + + +class MISPSharingGroup(AbstractMISP): + + def __init__(self): + super(MISPSharingGroup, self).__init__() + + def from_dict(self, **kwargs): + if kwargs.get('SharingGroup'): + kwargs = kwargs.get('SharingGroup') + super(MISPSharingGroup, self).from_dict(**kwargs) + class MISPLog(AbstractMISP): def __init__(self): super(MISPLog, self).__init__() + def from_dict(self, **kwargs): + if kwargs.get('Log'): + kwargs = kwargs.get('Log') + super(MISPLog, self).from_dict(**kwargs) + def __repr__(self): return '<{self.__class__.__name__}({self.model}, {self.action}, {self.title})'.format(self=self) @@ -855,7 +949,7 @@ class MISPSighting(AbstractMISP): def __init__(self): super(MISPSighting, self).__init__() - def from_dict(self, value=None, uuid=None, id=None, source=None, type=None, timestamp=None, **kwargs): + def from_dict(self, **kwargs): """Initialize the MISPSighting from a dictionary :value: Value of the attribute the sighting is related too. Pushing this object will update the sighting count of each attriutes with thifs value on the instance @@ -865,12 +959,8 @@ class MISPSighting(AbstractMISP): :type: Type of the sighting :timestamp: Timestamp associated to the sighting """ - self.value = value - self.uuid = uuid - self.id = id - self.source = source - self.type = type - self.timestamp = timestamp + if kwargs.get('Sighting'): + kwargs = kwargs.get('Sighting') super(MISPSighting, self).from_dict(**kwargs) def __repr__(self): @@ -917,7 +1007,6 @@ class MISPObjectAttribute(MISPAttribute): class MISPShadowAttribute(AbstractMISP): - # NOTE: Kindof a MISPAttribute, but can be lot more lightweight (just one key for example) def __init__(self): super(MISPShadowAttribute, self).__init__() @@ -927,6 +1016,11 @@ class MISPShadowAttribute(AbstractMISP): kwargs = kwargs.get('ShadowAttribute') super(MISPShadowAttribute, self).from_dict(**kwargs) + def __repr__(self): + if hasattr(self, 'value'): + return '<{self.__class__.__name__}(type={self.type}, value={self.value})'.format(self=self) + return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) + class MISPObject(AbstractMISP): @@ -1176,12 +1270,3 @@ class MISPObject(AbstractMISP): if hasattr(self, 'name'): return '<{self.__class__.__name__}(name={self.name})'.format(self=self) return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) - - -class MISPSharingGroup(AbstractMISP): - - def __init__(self): - super(MISPSharingGroup, self).__init__() - - def from_dict(self, **kwargs): - super(MISPSharingGroup, self).from_dict(**kwargs) diff --git a/setup.py b/setup.py index fe31f82..4a27b15 100644 --- a/setup.py +++ b/setup.py @@ -41,8 +41,8 @@ setup( ], install_requires=['six', 'requests', 'python-dateutil', 'jsonschema', 'python-dateutil', 'enum34;python_version<"3.4"', - 'functools32;python_version<"3.0"'], - extras_require={'fileobjects': ['lief>=0.8', 'python-magic', 'pydeep'], + 'functools32;python_version<"3.0"', 'deprecated'], + extras_require={'fileobjects': ['lief>=0.8,<0.10;python_version<"3.5"', 'lief>=0.10.0.dev0;python_version>"3.5"', 'python-magic', 'pydeep'], 'neo': ['py2neo'], 'openioc': ['beautifulsoup4'], 'virustotal': ['validators'], diff --git a/tests/testlive_comprehensive.py b/tests/testlive_comprehensive.py index 8a84318..e4fd526 100644 --- a/tests/testlive_comprehensive.py +++ b/tests/testlive_comprehensive.py @@ -13,6 +13,7 @@ import re import json from pathlib import Path +import urllib3 import time from uuid import uuid4 @@ -20,8 +21,8 @@ import logging logging.disable(logging.CRITICAL) try: - from pymisp import ExpandedPyMISP, MISPEvent, MISPOrganisation, MISPUser, Distribution, ThreatLevel, Analysis, MISPObject, MISPAttribute - from pymisp.tools import CSVLoader, DomainIPObject, ASNObject + from pymisp import ExpandedPyMISP, MISPEvent, MISPOrganisation, MISPUser, Distribution, ThreatLevel, Analysis, MISPObject, MISPAttribute, MISPSighting, MISPShadowAttribute, MISPTag, MISPSharingGroup, MISPFeed, MISPServer + from pymisp.tools import CSVLoader, DomainIPObject, ASNObject, GenericObjectGenerator except ImportError: if sys.version_info < (3, 6): print('This test suite requires Python 3.6+, breaking.') @@ -35,12 +36,15 @@ try: travis_run = True except ImportError as e: print(e) - url = 'http://localhost:8080' - key = 'HRizIMmaxBOXAQSzKZ874rDWUsQEk4vGAGBoljQO' + url = 'https://localhost:8443' + key = 'K5yV0CcxdnklzDfCKlnPniIxrMX41utQ2dG13zZ3' verifycert = False travis_run = False +urllib3.disable_warnings() + + class TestComprehensive(unittest.TestCase): @classmethod @@ -58,14 +62,14 @@ class TestComprehensive(unittest.TestCase): user = MISPUser() user.email = 'testusr@user.local' user.org_id = cls.test_org.id - cls.test_usr = cls.admin_misp_connector.add_user(user) + cls.test_usr = cls.admin_misp_connector.add_user(user, pythonify=True) cls.user_misp_connector = ExpandedPyMISP(url, cls.test_usr.authkey, verifycert, debug=False) # Creates a publisher user = MISPUser() user.email = 'testpub@user.local' user.org_id = cls.test_org.id user.role_id = 4 - cls.test_pub = cls.admin_misp_connector.add_user(user) + cls.test_pub = cls.admin_misp_connector.add_user(user, pythonify=True) cls.pub_misp_connector = ExpandedPyMISP(url, cls.test_pub.authkey, verifycert) # Update all json stuff cls.admin_misp_connector.update_object_templates() @@ -81,7 +85,7 @@ class TestComprehensive(unittest.TestCase): # Delete user cls.admin_misp_connector.delete_user(user_id=cls.test_usr.id) # Delete org - cls.admin_misp_connector.delete_organisation(org_id=cls.test_org.id) + cls.admin_misp_connector.delete_organisation(organisation_id=cls.test_org.id) def create_simple_event(self, force_timestamps=False): mispevent = MISPEvent(force_timestamps=force_timestamps) @@ -497,14 +501,13 @@ class TestComprehensive(unittest.TestCase): self.assertEqual(first.objects[1].distribution, Distribution.inherit.value) self.assertEqual(first.objects[1].attributes[0].distribution, Distribution.inherit.value) # Attribute create - attribute = self.user_misp_connector.add_named_attribute(first, 'comment', 'bar') - # FIXME: Add helper that returns a list of MISPAttribute - self.assertEqual(attribute[0]['Attribute']['distribution'], str(Distribution.inherit.value)) + attribute = self.user_misp_connector.add_attribute(first.id, {'type': 'comment', 'value': 'bar'}, pythonify=True) + self.assertEqual(attribute.value, 'bar', attribute.to_json()) + self.assertEqual(attribute.distribution, Distribution.inherit.value, attribute.to_json()) # Object - add o = MISPObject('file') o.add_attribute('filename', value='blah.exe') new_obj = self.user_misp_connector.add_object(first.id, o) - # FIXME: Add helper that returns a MISPObject self.assertEqual(new_obj.distribution, int(Distribution.inherit.value)) self.assertEqual(new_obj.attributes[0].distribution, int(Distribution.inherit.value)) # Object - edit @@ -699,6 +702,13 @@ class TestComprehensive(unittest.TestCase): self.assertEqual(len(events), 1) self.assertIs(events[0].attributes[-1].malware_binary, None) + # Search index + events = self.user_misp_connector.search_index(timestamp=first.timestamp.timestamp(), + pythonify=True) + self.assertEqual(len(events), 1) + self.assertEqual(events[0].info, 'foo bar blah') + self.assertEqual(events[0].attributes, []) + finally: # Delete event self.admin_misp_connector.delete_event(first.id) @@ -712,12 +722,12 @@ class TestComprehensive(unittest.TestCase): first.attributes[0].comment = 'This is the modified comment' attribute = self.user_misp_connector.update_attribute(first.attributes[0]) self.assertEqual(attribute.comment, 'This is the modified comment') - attribute = self.user_misp_connector.change_comment(first.attributes[0].uuid, 'This is the modified comment, again') - self.assertEqual(attribute['Attribute']['comment'], 'This is the modified comment, again') - attribute = self.user_misp_connector.change_disable_correlation(first.attributes[0].uuid, True) - self.assertEqual(attribute['Attribute']['disable_correlation'], True) - attribute = self.user_misp_connector.change_disable_correlation(first.attributes[0].uuid, 0) - self.assertEqual(attribute['Attribute']['disable_correlation'], False) + attribute = self.user_misp_connector.update_attribute({'comment': 'This is the modified comment, again'}, attribute.id) + self.assertEqual(attribute.comment, 'This is the modified comment, again') + attribute = self.user_misp_connector.update_attribute({'disable_correlation': True}, attribute.id) + self.assertTrue(attribute.disable_correlation) + attribute = self.user_misp_connector.update_attribute({'disable_correlation': False}, attribute.id) + self.assertFalse(attribute.disable_correlation) finally: # Delete event self.admin_misp_connector.delete_event(first.id) @@ -730,10 +740,15 @@ class TestComprehensive(unittest.TestCase): second = self.user_misp_connector.add_event(second) current_ts = int(time.time()) - self.user_misp_connector.sighting(value=first.attributes[0].value) - self.user_misp_connector.sighting(value=second.attributes[0].value, - source='Testcases', - type='1') + r = self.user_misp_connector.add_sighting({'value': first.attributes[0].value}) + self.assertEqual(r['message'], 'Sighting added') + + s = MISPSighting() + s.value = second.attributes[0].value + s.source = 'Testcases' + s.type = '1' + r = self.user_misp_connector.add_sighting(s, second.attributes[0].id) + self.assertEqual(r['message'], 'Sighting added') s = self.user_misp_connector.search_sightings(publish_timestamp=current_ts, include_attribute=True, include_event_meta=True, pythonify=True) @@ -747,7 +762,7 @@ class TestComprehensive(unittest.TestCase): include_event_meta=True, pythonify=True) self.assertEqual(len(s), 1) - self.assertEqual(s[0]['event'].id, second.id) + self.assertEqual(s[0]['event'].id, second.id, s) self.assertEqual(s[0]['attribute'].id, second.attributes[0].id) s = self.user_misp_connector.search_sightings(publish_timestamp=current_ts, @@ -770,6 +785,19 @@ class TestComprehensive(unittest.TestCase): pythonify=True) self.assertEqual(len(s), 1) self.assertEqual(s[0]['sighting'].attribute_id, str(second.attributes[0].id)) + + # Get sightings from event/attribute / org + s = self.user_misp_connector.sightings(first, pythonify=True) + self.assertTrue(isinstance(s, list)) + self.assertEqual(int(s[0].attribute_id), first.attributes[0].id) + + r = self.admin_misp_connector.add_sighting(s, second.attributes[0].id) + self.assertEqual(r['message'], 'Sighting added') + s = self.user_misp_connector.sightings(second.attributes[0], pythonify=True) + self.assertEqual(len(s), 2) + s = self.user_misp_connector.sightings(second.attributes[0], self.test_org.id, pythonify=True) + self.assertEqual(len(s), 1) + self.assertEqual(s[0].org_id, self.test_org.id) finally: # Delete event self.admin_misp_connector.delete_event(first.id) @@ -786,13 +814,13 @@ class TestComprehensive(unittest.TestCase): first = self.user_misp_connector.add_event(first) second = self.user_misp_connector.add_event(second) - response = self.user_misp_connector.fast_publish(first.id, alert=False) + response = self.user_misp_connector.publish(first.id, alert=False) self.assertEqual(response['errors'][1]['message'], 'You do not have permission to use this functionality.') # Default search, attribute with to_ids == True first.attributes[0].to_ids = True first = self.user_misp_connector.update_event(first) - self.admin_misp_connector.fast_publish(first.id, alert=False) + self.admin_misp_connector.publish(first.id, alert=False) csv = self.user_misp_connector.search(return_format='csv', publish_timestamp=first.timestamp.timestamp(), pythonify=True) self.assertEqual(len(csv), 1) self.assertEqual(csv[0]['value'], first.attributes[0].value) @@ -848,6 +876,9 @@ class TestComprehensive(unittest.TestCase): for k in columns: self.assertTrue(k in csv[0]) + # FIXME Publish is async, if we delete the event too fast, we have an empty one. + # https://github.com/MISP/MISP/issues/4886 + time.sleep(10) finally: # Delete event self.admin_misp_connector.delete_event(first.id) @@ -869,6 +900,7 @@ class TestComprehensive(unittest.TestCase): # Delete event self.admin_misp_connector.delete_event(first.id) + @unittest.skip("Wait for https://github.com/MISP/MISP/issues/4848") def test_upload_sample(self): first = self.create_simple_event() second = self.create_simple_event() @@ -876,11 +908,8 @@ class TestComprehensive(unittest.TestCase): try: # Simple, not executable first = self.user_misp_connector.add_event(first) - with open('tests/testlive_comprehensive.py', 'rb') as f: - response = self.user_misp_connector.upload_sample(filename='testfile.py', filepath_or_bytes=f.read(), - event_id=first.id) + response = self.user_misp_connector.add_sample_to_event(event_id=first.id, path_to_sample=Path('tests/testlive_comprehensive.py')) self.assertTrue('message' in response, "Content of response: {}".format(response)) - print(response) self.assertEqual(response['message'], 'Success, saved all attributes.') first = self.user_misp_connector.get_event(first.id) self.assertEqual(len(first.objects), 1) @@ -888,8 +917,8 @@ class TestComprehensive(unittest.TestCase): # Simple, executable second = self.user_misp_connector.add_event(second) with open('tests/viper-test-files/test_files/whoami.exe', 'rb') as f: - response = self.user_misp_connector.upload_sample(filename='whoami.exe', filepath_or_bytes=f.read(), - event_id=second.id) + pseudofile = BytesIO(f.read()) + response = self.user_misp_connector.add_sample_to_event(event_id=second.id, filename='whoami.exe', pseudofile=pseudofile) self.assertEqual(response['message'], 'Success, saved all attributes.') second = self.user_misp_connector.get_event(second.id) self.assertEqual(len(second.objects), 1) @@ -897,9 +926,7 @@ class TestComprehensive(unittest.TestCase): third = self.user_misp_connector.add_event(third) if not travis_run: # Advanced, executable - with open('tests/viper-test-files/test_files/whoami.exe', 'rb') as f: - response = self.user_misp_connector.upload_sample(filename='whoami.exe', filepath_or_bytes=f.read(), - event_id=third.id, advanced_extraction=True) + response = self.user_misp_connector.add_sample_to_event(event_id=third.id, path_to_sample=Path('tests/viper-test-files/test_files/whoami.exe'), advanced_extraction=True) self.assertEqual(response['message'], 'Success, saved all attributes.') third = self.user_misp_connector.get_event(third.id) self.assertEqual(len(third.objects), 7) @@ -932,19 +959,67 @@ class TestComprehensive(unittest.TestCase): # Test with add_attributes second = self.create_simple_event() ip_dom = MISPObject('domain-ip') - ip_dom.add_attribute('domain', value='google.fr') + ip_dom.add_attribute('domain', value='google.fr', disable_correlation=True) ip_dom.add_attributes('ip', {'value': '10.8.8.8', 'to_ids': False}, '10.9.8.8') ip_dom.add_attributes('ip', '11.8.8.8', '11.9.8.8') second.add_object(ip_dom) second = self.user_misp_connector.add_event(second) self.assertEqual(len(second.objects[0].attributes), 5) + self.assertTrue(second.objects[0].attributes[0].disable_correlation) self.assertFalse(second.objects[0].attributes[1].to_ids) self.assertTrue(second.objects[0].attributes[2].to_ids) + + # Test generic Tag methods + r = self.admin_misp_connector.tag(second, 'generic_tag_test') + self.assertTrue(r['message'].endswith(f'successfully attached to Event({second.id}).'), r['message']) + r = self.admin_misp_connector.untag(second, 'generic_tag_test') + self.assertTrue(r['message'].endswith(f'successfully removed from Event({second.id}).'), r['message']) + # NOTE: object tagging not supported yet + # r = self.admin_misp_connector.tag(second.objects[0].uuid, 'generic_tag_test') + # self.assertTrue(r['message'].endswith(f'successfully attached to Object({second.objects[0].id}).'), r['message']) + # r = self.admin_misp_connector.untag(second.objects[0].uuid, 'generic_tag_test') + # self.assertTrue(r['message'].endswith(f'successfully removed from Object({second.objects[0].id}).'), r['message']) + r = self.admin_misp_connector.tag(second.objects[0].attributes[0].uuid, 'generic_tag_test') + self.assertTrue(r['message'].endswith(f'successfully attached to Attribute({second.objects[0].attributes[0].id}).'), r['message']) + r = self.admin_misp_connector.untag(second.objects[0].attributes[0].uuid, 'generic_tag_test') + self.assertTrue(r['message'].endswith(f'successfully removed from Attribute({second.objects[0].attributes[0].id}).'), r['message']) + + # Delete tag to avoid polluting the db + tags = self.admin_misp_connector.tags(pythonify=True) + for t in tags: + if t.name == 'generic_tag_test': + response = self.admin_misp_connector.delete_tag(t.id) + self.assertEqual(response['message'], 'Tag deleted.') + + # Test delete object + r = self.user_misp_connector.delete_object(second.objects[0].id) + self.assertEqual(r['message'], 'Object deleted') finally: # Delete event self.admin_misp_connector.delete_event(first.id) self.admin_misp_connector.delete_event(second.id) + def test_unknown_template(self): + first = self.create_simple_event() + attributeAsDict = [{'MyCoolAttribute': {'value': 'critical thing', 'type': 'text'}}, + {'MyCoolerAttribute': {'value': 'even worse', 'type': 'text', 'disable_correlation': True}}] + misp_object = GenericObjectGenerator('my-cool-template') + misp_object.generate_attributes(attributeAsDict) + first.add_object(misp_object) + blah_object = MISPObject('BLAH_TEST') + blah_object.add_reference(misp_object.uuid, "test relation") + blah_object.add_attribute('transaction-number', value='foo', type="text", disable_correlation=True) + first.add_object(blah_object) + try: + first = self.user_misp_connector.add_event(first) + self.assertEqual(len(first.objects[0].attributes), 2) + self.assertFalse(first.objects[0].attributes[0].disable_correlation) + self.assertTrue(first.objects[0].attributes[1].disable_correlation) + self.assertTrue(first.objects[1].attributes[0].disable_correlation) + finally: + # Delete event + self.admin_misp_connector.delete_event(first.id) + def test_domain_ip_object(self): first = self.create_simple_event() try: @@ -975,24 +1050,38 @@ class TestComprehensive(unittest.TestCase): def test_object_template(self): r = self.admin_misp_connector.update_object_templates() self.assertEqual(type(r), list) - if not travis_run: - template = self.admin_misp_connector.get_object_template('688c46fb-5edb-40a3-8273-1af7923e2215') - self.assertEqual(template['ObjectTemplate']['uuid'], '688c46fb-5edb-40a3-8273-1af7923e2215') + object_templates = self.admin_misp_connector.object_templates(pythonify=True) + self.assertTrue(isinstance(object_templates, list)) + for object_template in object_templates: + if object_template.name == 'file': + break + + template = self.admin_misp_connector.get_object_template(object_template.uuid, pythonify=True) + self.assertEqual(template.name, 'file') def test_tags(self): # Get list - tags = self.admin_misp_connector.get_tags_list() + tags = self.admin_misp_connector.tags(pythonify=True) self.assertTrue(isinstance(tags, list)) # Get tag for tag in tags: - if not tag['hide_tag']: + if not tag.hide_tag: break - tag = self.admin_misp_connector.get_tag(tags[0]['id']) + tag = self.admin_misp_connector.get_tag(tag.id, pythonify=True) self.assertTrue('name' in tag) - r = self.admin_misp_connector.disable_tag(tag['id']) - self.assertTrue(r['Tag']['hide_tag']) - r = self.admin_misp_connector.enable_tag(tag['id']) - self.assertFalse(r['Tag']['hide_tag']) + # Enable by MISPTag + tag = self.admin_misp_connector.disable_tag(tag, pythonify=True) + self.assertTrue(tag.hide_tag) + tag = self.admin_misp_connector.enable_tag(tag, pythonify=True) + self.assertFalse(tag.hide_tag) + # Add tag + tag = MISPTag() + tag.name = 'this is a test tag' + new_tag = self.admin_misp_connector.add_tag(tag, pythonify=True) + self.assertEqual(new_tag.name, tag.name) + # Delete tag + response = self.admin_misp_connector.delete_tag(new_tag.id) + self.assertEqual(response['message'], 'Tag deleted.') def test_add_event_with_attachment_object_controller(self): first = self.create_simple_event() @@ -1006,16 +1095,20 @@ class TestComprehensive(unittest.TestCase): r = self.user_misp_connector.add_object(first.id, peo) self.assertEqual(r.name, 'pe', r) for ref in peo.ObjectReference: - r = self.user_misp_connector.add_object_reference(ref) - self.assertTrue('ObjectReference' in r, r) + r = self.user_misp_connector.add_object_reference(ref, pythonify=True) + # FIXME: https://github.com/MISP/MISP/issues/4866 + # self.assertEqual(r.object_uuid, peo.uuid, r.to_json()) r = self.user_misp_connector.add_object(first.id, fo) obj_attrs = r.get_attributes_by_relation('ssdeep') self.assertEqual(len(obj_attrs), 1, obj_attrs) self.assertEqual(r.name, 'file', r) - for ref in fo.ObjectReference: - r = self.user_misp_connector.add_object_reference(ref) - self.assertTrue('ObjectReference' in r, r) + r = self.user_misp_connector.add_object_reference(fo.ObjectReference[0], pythonify=True) + # FIXME: https://github.com/MISP/MISP/issues/4866 + # self.assertEqual(r.object_uuid, fo.uuid, r.to_json()) + self.assertEqual(r.referenced_uuid, peo.uuid, r.to_json()) + r = self.user_misp_connector.delete_object_reference(r.id) + self.assertEqual(r['message'], 'ObjectReference deleted') finally: # Delete event self.admin_misp_connector.delete_event(first.id) @@ -1043,19 +1136,22 @@ class TestComprehensive(unittest.TestCase): r = self.admin_misp_connector.update_taxonomies() self.assertEqual(r['name'], 'All taxonomy libraries are up to date already.') # Get list - taxonomies = self.admin_misp_connector.get_taxonomies_list() + taxonomies = self.admin_misp_connector.taxonomies(pythonify=True) self.assertTrue(isinstance(taxonomies, list)) list_name_test = 'tlp' for tax in taxonomies: - if tax['Taxonomy']['namespace'] == list_name_test: + if tax.namespace == list_name_test: break if not travis_run: - r = self.admin_misp_connector.get_taxonomy(tax['Taxonomy']['id']) - self.assertEqual(r['Taxonomy']['namespace'], list_name_test) - self.assertTrue('enabled' in r['Taxonomy']) - r = self.admin_misp_connector.enable_taxonomy(tax['Taxonomy']['id']) + r = self.admin_misp_connector.get_taxonomy(tax.id, pythonify=True) + self.assertEqual(r.namespace, list_name_test) + self.assertTrue('enabled' in r) + r = self.admin_misp_connector.enable_taxonomy(tax.id) self.assertEqual(r['message'], 'Taxonomy enabled') - r = self.admin_misp_connector.disable_taxonomy(tax['Taxonomy']['id']) + r = self.admin_misp_connector.enable_taxonomy_tags(tax.id) + # FIXME: https://github.com/MISP/MISP/issues/4865 + # self.assertEqual(r, []) + r = self.admin_misp_connector.disable_taxonomy(tax.id) self.assertEqual(r['message'], 'Taxonomy disabled') def test_warninglists(self): @@ -1067,21 +1163,24 @@ class TestComprehensive(unittest.TestCase): except Exception: print(r) # Get list - r = self.admin_misp_connector.get_warninglists() - # FIXME It returns Warninglists object instead of a list of warning lists directly. This is inconsistent. - warninglists = r['Warninglists'] + warninglists = self.admin_misp_connector.warninglists(pythonify=True) self.assertTrue(isinstance(warninglists, list)) list_name_test = 'List of known hashes with common false-positives (based on Florian Roth input list)' for wl in warninglists: - if wl['Warninglist']['name'] == list_name_test: + if wl.name == list_name_test: break - testwl = wl['Warninglist'] - r = self.admin_misp_connector.get_warninglist(testwl['id']) - self.assertEqual(r['Warninglist']['name'], list_name_test) - self.assertTrue('WarninglistEntry' in r['Warninglist']) - r = self.admin_misp_connector.enable_warninglist(testwl['id']) + testwl = wl + r = self.admin_misp_connector.get_warninglist(testwl.id, pythonify=True) + self.assertEqual(r.name, list_name_test) + self.assertTrue('WarninglistEntry' in r) + r = self.admin_misp_connector.enable_warninglist(testwl.id) self.assertEqual(r['success'], '1 warninglist(s) enabled') - r = self.admin_misp_connector.disable_warninglist(testwl['id']) + # Check if a value is in a warning list + md5_empty_file = 'd41d8cd98f00b204e9800998ecf8427e' + r = self.user_misp_connector.values_in_warninglist([md5_empty_file]) + self.assertEqual(r[md5_empty_file][0]['name'], list_name_test) + + r = self.admin_misp_connector.disable_warninglist(testwl.id) self.assertEqual(r['success'], '1 warninglist(s) disabled') def test_noticelists(self): @@ -1089,20 +1188,21 @@ class TestComprehensive(unittest.TestCase): r = self.admin_misp_connector.update_noticelists() self.assertEqual(r['name'], 'All noticelists are up to date already.') # Get list - noticelists = self.admin_misp_connector.get_noticelists() + noticelists = self.admin_misp_connector.noticelists(pythonify=True) self.assertTrue(isinstance(noticelists, list)) list_name_test = 'gdpr' for nl in noticelists: - if nl['Noticelist']['name'] == list_name_test: + if nl.name == list_name_test: break testnl = nl - r = self.admin_misp_connector.get_noticelist(testnl['Noticelist']['id']) - self.assertEqual(r['Noticelist']['name'], list_name_test) - self.assertTrue('NoticelistEntry' in r['Noticelist']) - r = self.admin_misp_connector.enable_noticelist(testnl['Noticelist']['id']) - self.assertTrue(r['Noticelist']['enabled']) - r = self.admin_misp_connector.disable_noticelist(testnl['Noticelist']['id']) - self.assertFalse(r['Noticelist']['enabled']) + r = self.admin_misp_connector.get_noticelist(testnl.id, pythonify=True) + self.assertEqual(r.name, list_name_test) + # FIXME: https://github.com/MISP/MISP/issues/4856 + self.assertTrue('NoticelistEntry' in r) + r = self.admin_misp_connector.enable_noticelist(testnl.id) + self.assertTrue(r['Noticelist']['enabled'], r) + r = self.admin_misp_connector.disable_noticelist(testnl.id) + self.assertFalse(r['Noticelist']['enabled'], r) def test_galaxies(self): if not travis_run: @@ -1110,22 +1210,23 @@ class TestComprehensive(unittest.TestCase): r = self.admin_misp_connector.update_galaxies() self.assertEqual(r['name'], 'Galaxies updated.') # Get list - galaxies = self.admin_misp_connector.get_galaxies() + galaxies = self.admin_misp_connector.galaxies(pythonify=True) self.assertTrue(isinstance(galaxies, list)) list_name_test = 'Mobile Attack - Attack Pattern' for galaxy in galaxies: - if galaxy['Galaxy']['name'] == list_name_test: + if galaxy.name == list_name_test: break - r = self.admin_misp_connector.get_galaxy(galaxy['Galaxy']['id']) - self.assertEqual(r['Galaxy']['name'], list_name_test) - self.assertTrue('GalaxyCluster' in r) + r = self.admin_misp_connector.get_galaxy(galaxy.id, pythonify=True) + self.assertEqual(r.name, list_name_test) + # FIXME: Fails due to https://github.com/MISP/MISP/issues/4855 + # self.assertTrue('GalaxyCluster' in r) def test_zmq(self): first = self.create_simple_event() try: first = self.user_misp_connector.add_event(first) if not travis_run: - r = self.admin_misp_connector.pushEventToZMQ(first.id) + r = self.admin_misp_connector.push_event_to_ZMQ(first.id) self.assertEqual(r['message'], 'Event published to ZMQ') finally: # Delete event @@ -1150,11 +1251,44 @@ class TestComprehensive(unittest.TestCase): self.admin_misp_connector.delete_event(first.id) def test_user(self): - user = self.user_misp_connector.get_user() + # Get list + users = self.admin_misp_connector.users(pythonify=True) + self.assertTrue(isinstance(users, list)) + users_email = 'testusr@user.local' + for user in users: + if user.email == users_email: + break + self.assertEqual(user.email, users_email) + # get user + user = self.user_misp_connector.get_user(pythonify=True) self.assertEqual(user.authkey, self.test_usr.authkey) + # Update user + user.email = 'foo@bar.de' + user = self.admin_misp_connector.update_user(user, pythonify=True) + self.assertEqual(user.email, 'foo@bar.de') + + def test_organisation(self): + # Get list + orgs = self.admin_misp_connector.organisations(pythonify=True) + self.assertTrue(isinstance(orgs, list)) + org_name = 'ORGNAME' + for org in orgs: + if org.name == org_name: + break + self.assertEqual(org.name, org_name) + # Get org + organisation = self.user_misp_connector.get_organisation(self.test_usr.org_id) + self.assertEqual(organisation.name, 'Test Org') + # Update org + organisation.name = 'blah' + organisation = self.admin_misp_connector.update_organisation(organisation) + self.assertEqual(organisation.name, 'blah') def test_attribute(self): first = self.create_simple_event() + second = self.create_simple_event() + second.add_attribute('ip-src', '11.11.11.11') + second.distribution = Distribution.all_communities try: first = self.user_misp_connector.add_event(first) # Get attribute @@ -1178,8 +1312,11 @@ class TestComprehensive(unittest.TestCase): new_attribute = self.user_misp_connector.update_attribute(new_attribute) self.assertEqual(new_attribute.value, '5.6.3.4') # Update attribute as proposal - new_proposal_update = self.user_misp_connector.update_attribute_proposal(new_attribute.id, {'to_ids': False}) + new_proposal_update = self.user_misp_connector.update_attribute_proposal(new_attribute.id, {'to_ids': False}, pythonify=True) self.assertEqual(new_proposal_update.to_ids, False) + # Delete attribute as proposal + proposal_delete = self.user_misp_connector.delete_attribute_proposal(new_attribute.id) + self.assertTrue(proposal_delete['saved']) # Get attribute proposal temp_new_proposal = self.user_misp_connector.get_attribute_proposal(new_proposal.id) self.assertEqual(temp_new_proposal.uuid, new_proposal.uuid) @@ -1198,35 +1335,288 @@ class TestComprehensive(unittest.TestCase): self.assertEqual(response['message'], 'Proposal discarded.') attribute = self.user_misp_connector.get_attribute(new_attribute.id) self.assertEqual(attribute.to_ids, False) + + # Test fallback to proposal if the user doesn't own the event + second = self.admin_misp_connector.add_event(second, pythonify=True) + # FIXME: attribute needs to be a complete MISPAttribute: https://github.com/MISP/MISP/issues/4868 + prop_attr = MISPAttribute() + prop_attr.from_dict(**{'type': 'ip-dst', 'value': '123.43.32.21'}) + attribute = self.user_misp_connector.add_attribute(second.id, prop_attr) + self.assertTrue(isinstance(attribute, MISPShadowAttribute)) + attribute = self.user_misp_connector.update_attribute({'comment': 'blah'}, second.attributes[0].id) + self.assertTrue(isinstance(attribute, MISPShadowAttribute)) + self.assertEqual(attribute.value, second.attributes[0].value) + response = self.user_misp_connector.delete_attribute(second.attributes[1].id) + self.assertTrue(response['success']) + response = self.admin_misp_connector.delete_attribute(second.attributes[1].id) + self.assertEqual(response['message'], 'Attribute deleted.') finally: # Delete event self.admin_misp_connector.delete_event(first.id) + self.admin_misp_connector.delete_event(second.id) - @unittest.skip("Currently failing") def test_search_type_event_csv(self): try: first, second, third = self.environment() # Search as admin - events = self.admin_misp_connector.search(return_format='csv', timestamp=first.timestamp.timestamp()) - print(events) + events = self.admin_misp_connector.search(return_format='csv', timestamp=first.timestamp.timestamp(), pythonify=True) + self.assertTrue(isinstance(events, list)) + self.assertEqual(len(events), 8) attributes_types_search = self.admin_misp_connector.build_complex_query(or_parameters=['ip-src', 'ip-dst']) events = self.admin_misp_connector.search(return_format='csv', timestamp=first.timestamp.timestamp(), - type_attribute=attributes_types_search) - print(events) + type_attribute=attributes_types_search, pythonify=True) + self.assertTrue(isinstance(events, list)) + self.assertEqual(len(events), 6) finally: # Delete event self.admin_misp_connector.delete_event(first.id) self.admin_misp_connector.delete_event(second.id) self.admin_misp_connector.delete_event(third.id) + def test_search_logs(self): + # FIXME: https://github.com/MISP/MISP/issues/4872 + r = self.admin_misp_connector.search_logs(model='User', created=date.today(), pythonify=True) + for entry in r[-2:]: + self.assertEqual(entry.action, 'add') + def test_live_acl(self): - missing_acls = self.admin_misp_connector.get_live_query_acl() + missing_acls = self.admin_misp_connector.remote_acl self.assertEqual(missing_acls, [], msg=missing_acls) def test_roles(self): role = self.admin_misp_connector.set_default_role(4) self.assertEqual(role['message'], 'Default role set.') self.admin_misp_connector.set_default_role(3) + roles = self.admin_misp_connector.roles(pythonify=True) + self.assertTrue(isinstance(roles, list)) + + def test_describe_types(self): + remote = self.admin_misp_connector.describe_types_remote + local = self.admin_misp_connector.describe_types_local + self.assertDictEqual(remote, local) + + def test_versions(self): + self.assertEqual(self.user_misp_connector.version, self.user_misp_connector.pymisp_version_master) + self.assertEqual(self.user_misp_connector.misp_instance_version['version'], + self.user_misp_connector.misp_instance_version_master['version']) + + def test_statistics(self): + try: + # Attributes + first, second, third = self.environment() + expected_attr_stats = {'ip-dst': '2', 'ip-src': '1', 'text': '5'} + attr_stats = self.admin_misp_connector.attributes_statistics() + self.assertDictEqual(attr_stats, expected_attr_stats) + expected_attr_stats_percent = {'ip-dst': '25%', 'ip-src': '12.5%', 'text': '62.5%'} + attr_stats = self.admin_misp_connector.attributes_statistics(percentage=True) + self.assertDictEqual(attr_stats, expected_attr_stats_percent) + expected_attr_stats_category_percent = {'Network activity': '37.5%', 'Other': '62.5%'} + attr_stats = self.admin_misp_connector.attributes_statistics(context='category', percentage=True) + self.assertDictEqual(attr_stats, expected_attr_stats_category_percent) + # Tags + to_test = {'tags': {'tlp:white___test': '1'}, 'taxonomies': {'workflow': 0}} + tags_stats = self.admin_misp_connector.tags_statistics() + self.assertDictEqual(tags_stats, to_test) + to_test = {'tags': {'tlp:white___test': '100%'}, 'taxonomies': {'workflow': '0%'}} + tags_stats = self.admin_misp_connector.tags_statistics(percentage=True, name_sort=True) + self.assertDictEqual(tags_stats, to_test) + # Users + to_test = {'stats': {'event_count': 3, 'event_count_month': 3, 'attribute_count': 8, + 'attribute_count_month': 8, 'attributes_per_event': 3, 'correlation_count': 1, + 'proposal_count': 0, 'user_count': 3, 'user_count_pgp': 0, 'org_count': 2, + 'local_org_count': 2, 'average_user_per_org': 1.5, 'thread_count': 0, + 'thread_count_month': 0, 'post_count': 0, 'post_count_month': 0}} + users_stats = self.admin_misp_connector.users_statistics(context='data') + self.assertDictEqual(users_stats, to_test) + + users_stats = self.admin_misp_connector.users_statistics(context='orgs') + self.assertTrue('ORGNAME' in list(users_stats.keys())) + + users_stats = self.admin_misp_connector.users_statistics(context='users') + self.assertEqual(list(users_stats.keys()), ['user', 'org_local', 'org_external']) + + users_stats = self.admin_misp_connector.users_statistics(context='tags') + self.assertEqual(list(users_stats.keys()), ['flatData', 'treemap']) + + # FIXME: https://github.com/MISP/MISP/issues/4880 + # users_stats = self.admin_misp_connector.users_statistics(context='attributehistogram') + + self.user_misp_connector.add_sighting({'value': first.attributes[0].value}) + users_stats = self.user_misp_connector.users_statistics(context='sightings') + self.assertEqual(list(users_stats.keys()), ['toplist', 'eventids']) + + users_stats = self.admin_misp_connector.users_statistics(context='galaxyMatrix') + self.assertTrue('matrix' in users_stats) + finally: + # Delete event + self.admin_misp_connector.delete_event(first.id) + self.admin_misp_connector.delete_event(second.id) + self.admin_misp_connector.delete_event(third.id) + + def test_direct(self): + try: + r = self.user_misp_connector.direct_call('events/add', data={'info': 'foo'}) + event = MISPEvent() + event.from_dict(**r) + r = self.user_misp_connector.direct_call(f'events/view/{event.id}') + event_get = MISPEvent() + event_get.from_dict(**r) + self.assertDictEqual(event.to_dict(), event_get.to_dict()) + finally: + self.admin_misp_connector.delete_event(event.id) + + def test_freetext(self): + first = self.create_simple_event() + try: + self.admin_misp_connector.toggle_warninglist(warninglist_name='%dns resolv%', force_enable=True) + first = self.user_misp_connector.add_event(first) + r = self.user_misp_connector.freetext(first.id, '1.1.1.1 foo@bar.de', adhereToWarninglists=False, + distribution=2, returnMetaAttributes=False, pythonify=True) + self.assertTrue(isinstance(r, list)) + self.assertEqual(r[0].value, '1.1.1.1') + + # FIXME: https://github.com/MISP/MISP/issues/4881 + # r_wl = self.user_misp_connector.freetext(first.id, '8.8.8.8 foo@bar.de', adhereToWarninglists=True, + # distribution=2, returnMetaAttributes=False) + # print(r_wl) + r = self.user_misp_connector.freetext(first.id, '1.1.1.1 foo@bar.de', adhereToWarninglists=True, + distribution=2, returnMetaAttributes=True) + self.assertTrue(isinstance(r, list)) + self.assertTrue(isinstance(r[0]['types'], dict)) + # NOTE: required, or the attributes are inserted *after* the event is deleted + # FIXME: https://github.com/MISP/MISP/issues/4886 + time.sleep(10) + finally: + # Delete event + self.admin_misp_connector.delete_event(first.id) + + def test_sharing_groups(self): + # add + sg = MISPSharingGroup() + sg.name = 'Testcases SG' + sg.releasability = 'Testing' + sharing_group = self.admin_misp_connector.add_sharing_group(sg, pythonify=True) + self.assertEqual(sharing_group.name, 'Testcases SG') + self.assertEqual(sharing_group.releasability, 'Testing') + # add org + # FIXME: https://github.com/MISP/MISP/issues/4884 + # r = self.admin_misp_connector.add_org_to_sharing_group(sharing_group.id, + # organisation_id=self.test_org.id, extend=True) + + # delete org + # FIXME: https://github.com/MISP/MISP/issues/4884 + # r = self.admin_misp_connector.remove_org_from_sharing_group(sharing_group.id, + # organisation_id=self.test_org.id) + + # Get list + sharing_groups = self.admin_misp_connector.sharing_groups(pythonify=True) + self.assertTrue(isinstance(sharing_groups, list)) + self.assertEqual(sharing_groups[0].name, 'Testcases SG') + + # Use the SG + + first = self.create_simple_event() + try: + first = self.user_misp_connector.add_event(first) + first = self.admin_misp_connector.change_sharing_group_on_entity(first, sharing_group.id) + self.assertEqual(first.SharingGroup['name'], 'Testcases SG') + # FIXME https://github.com/MISP/MISP/issues/4891 + # first_attribute = self.admin_misp_connector.change_sharing_group_on_entity(first.attributes[0], sharing_group.id) + # self.assertEqual(first_attribute.SharingGroup['name'], 'Testcases SG') + finally: + # Delete event + self.admin_misp_connector.delete_event(first.id) + + # delete + r = self.admin_misp_connector.delete_sharing_group(sharing_group.id) + self.assertEqual(r['message'], 'SharingGroup deleted') + + def test_feeds(self): + # Add + feed = MISPFeed() + feed.name = 'TestFeed' + feed.provider = 'TestFeed - Provider' + feed.url = 'http://example.com' + feed = self.admin_misp_connector.add_feed(feed, pythonify=True) + self.assertEqual(feed.name, 'TestFeed') + self.assertEqual(feed.url, 'http://example.com') + # Update + feed.name = 'TestFeed - Update' + feed = self.admin_misp_connector.update_feed(feed, pythonify=True) + self.assertEqual(feed.name, 'TestFeed - Update') + # Delete + r = self.admin_misp_connector.delete_feed(feed.id) + self.assertEqual(r['message'], 'Feed deleted.') + # List + feeds = self.admin_misp_connector.feeds(pythonify=True) + self.assertTrue(isinstance(feeds, list)) + for feed in feeds: + if feed.name == 'The Botvrij.eu Data': + break + # Get + botvrij = self.admin_misp_connector.get_feed(feed.id, pythonify=True) + self.assertEqual(botvrij.url, "http://www.botvrij.eu/data/feed-osint") + # Enable + # MISP OSINT + feed = self.admin_misp_connector.enable_feed(feeds[0].id, pythonify=True) + self.assertTrue(feed.enabled) + feed = self.admin_misp_connector.enable_feed_cache(feeds[0].id, pythonify=True) + self.assertTrue(feed.caching_enabled) + # Botvrij.eu + feed = self.admin_misp_connector.enable_feed(botvrij.id, pythonify=True) + self.assertTrue(feed.enabled) + feed = self.admin_misp_connector.enable_feed_cache(botvrij.id, pythonify=True) + self.assertTrue(feed.caching_enabled) + # Cache + r = self.admin_misp_connector.cache_feed(botvrij.id) + self.assertEqual(r['message'], 'Feed caching job initiated.') + # Fetch + # Cannot test that, it fetches all the events. + # r = self.admin_misp_connector.fetch_feed(botvrij.id) + # FIXME https://github.com/MISP/MISP/issues/4834#issuecomment-511889274 + # self.assertEqual(r['message'], 'Feed caching job initiated.') + + # Cache all enabled feeds + r = self.admin_misp_connector.cache_all_feeds() + self.assertEqual(r['message'], 'Feed caching job initiated.') + # Compare all enabled feeds + r = self.admin_misp_connector.compare_feeds() + # FIXME: https://github.com/MISP/MISP/issues/4834#issuecomment-511890466 + # self.assertEqual(r['message'], 'Feed caching job initiated.') + time.sleep(30) + # Disable both feeds + feed = self.admin_misp_connector.disable_feed(feeds[0].id, pythonify=True) + self.assertFalse(feed.enabled) + feed = self.admin_misp_connector.disable_feed(botvrij.id, pythonify=True) + self.assertFalse(feed.enabled) + feed = self.admin_misp_connector.disable_feed_cache(feeds[0].id, pythonify=True) + self.assertFalse(feed.enabled) + feed = self.admin_misp_connector.disable_feed_cache(botvrij.id, pythonify=True) + self.assertFalse(feed.enabled) + + def test_servers(self): + # add + server = MISPServer() + server.name = 'Test Server' + server.url = 'https://127.0.0.1' + server.remote_org_id = 1 + server.authkey = key + server = self.admin_misp_connector.add_server(server, pythonify=True) + self.assertEqual(server.name, 'Test Server') + # Update + server.name = 'Updated name' + server = self.admin_misp_connector.update_server(server, pythonify=True) + self.assertEqual(server.name, 'Updated name') + # List + servers = self.admin_misp_connector.servers(pythonify=True) + self.assertEqual(servers[0].name, 'Updated name') + # Delete + server = self.admin_misp_connector.delete_server(server.id) + # FIXME: https://github.com/MISP/MISP/issues/4889 + + def test_upload_stix(self): + # FIXME https://github.com/MISP/MISP/issues/4892 + pass if __name__ == '__main__':