diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 2e7a0c8..d8c1dc2 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -2,6 +2,103 @@ Changelog ========= +v2.4.99 (2018-12-06) +-------------------- + +New +~~~ +- Auto generate doc for PyMISPExpanded. [Raphaël Vinot] +- Search_index in ExpandedPyMISP, cleanup, update jupyter. [Raphaël + Vinot] +- Add log search. [Raphaël Vinot] +- Add test for pushing an event to ZMQ. [Raphaël Vinot] +- Change_distribution method. [Raphaël Vinot] +- Add test cases for sightings, cleanup. [Raphaël Vinot] +- [example] Added sighting rest search example. [Sami Mokaddem] +- [sighting] Added support of sighting REST API. [Sami Mokaddem] +- Allow to pass csv to return_format in search. [Raphaël Vinot] +- Page/limit in search. [Raphaël Vinot] + +Changes +~~~~~~~ +- Bump Changelog. [Raphaël Vinot] +- Bump version. [Raphaël Vinot] +- Bump misp-objects & describeTypes. [Raphaël Vinot] +- Bump Changelog. [Raphaël Vinot] +- Version bump. [Raphaël Vinot] +- Bump misp-objects. [Raphaël Vinot] +- Add test cases for default distribution levels. [Raphaël Vinot] +- Include proposals in attributes search. [Dawid Czarnecki] + + Add includeProposals param to the search method +- Bump misp-objects. [Raphaël Vinot] +- Update readme to document testing. [Raphaël Vinot] +- Fixes & update Jupyter. [Raphaël Vinot] +- [tuto] Update search. [Raphaël Vinot] +- Add a script to load the API key from the file system (training VM) + [Raphaël Vinot] +- Bump misp-objects. [Raphaël Vinot] +- Add print in testlive to debug travis. [Raphaël Vinot] +- Bump objects. [Raphaël Vinot] + +Fix +~~~ +- Auto generate doc for PyMISPExpanded. [Raphaël Vinot] +- Test failing on travis... [Raphaël Vinot] +- Properly handle errors on event creation/update. [Raphaël Vinot] +- Test case. [Raphaël Vinot] +- Do not run the zmq test on travis. [Raphaël Vinot] +- Type of quick_filter. [Raphaël Vinot] +- Quick_filter was broken. [Raphaël Vinot] +- Properly initialize the config when jupyter runs on the VM. [Raphaël + Vinot] +- Travis run. [Raphaël Vinot] +- Readme update + python3 + pep8. [Christophe Vandeplas] + + align python path to readme specifying python3 +- Feed-generator gitignore. [Christophe Vandeplas] +- Test cases. [Raphaël Vinot] + +Other +~~~~~ +- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot] +- Merge pull request #310 from DragonDev1906/master. [Raphaël Vinot] + + Added get_object & get_attribute (by ID) +- Dded get_object & get_attribute. [DragonDev1906] +- Merge pull request #307 from garanews/patch-1. [Raphaël Vinot] + + fix for last pymisp version +- Fix for last pymisp version. [garanews] +- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot] +- Merge pull request #305 from dawid- + czarnecki/feature/include_proposals. [Raphaël Vinot] + + chg: Include proposals in attributes search +- Merge pull request #301 from deralexxx/patch-7. [Raphaël Vinot] + + mention virtualenv +- Mention virtualenv. [Alexander J] + + mide make sense for people who want to use it with virtualenv +- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot] +- Be more precise with the supported time indicators. [Sascha + Rommelfangen] +- Fixed documentation bug. [Sascha Rommelfangen] +- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot] +- Merge pull request #295 from 3c7/fix/search_index_date. [Raphaël + Vinot] + + Fixes date parameters for search_index() function +- Fixes date parameters for search_index() function. [Nils Kuhnert] +- Merge branch 'sightingAPI' [Raphaël Vinot] +- Merge branch 'master' into sightingAPI. [Raphaël Vinot] +- Merge pull request #285 from juju4/devel. [Raphaël Vinot] + + align examples on custom usage of misp_verifycert +- Align examples on custom usage of misp_verifycert. [juju4] + + v2.4.96 (2018-10-12) -------------------- @@ -22,6 +119,7 @@ New Changes ~~~~~~~ +- Bump changelog. [Raphaël Vinot] - Bump version. [Raphaël Vinot] - Bump misp-objects. [Raphaël Vinot] - Allow to pass a json string to direct_call. [Raphaël Vinot] @@ -32,6 +130,7 @@ Changes Fix ~~~ +- Test cases sample files. [Raphaël Vinot] - Prevent checking length on a integer. [Sami Mokaddem] - Direct call & add example. [Raphaël Vinot] - Disable test for travis, take 2. [Raphaël Vinot] diff --git a/docs/source/modules.rst b/docs/source/modules.rst index 7db7414..5fc2210 100644 --- a/docs/source/modules.rst +++ b/docs/source/modules.rst @@ -14,6 +14,12 @@ PyMISP .. autoclass:: PyMISP :members: +PyMISPExpanded (Python 3.6+ only) +--------------------------------- + +.. autoclass:: PyMISPExpanded + :members: + MISPAbstract ------------ diff --git a/examples/add_generic_object.py b/examples/add_generic_object.py index 36e04cd..86a7675 100755 --- a/examples/add_generic_object.py +++ b/examples/add_generic_object.py @@ -20,10 +20,17 @@ if __name__ == '__main__': args = parser.parse_args() pymisp = PyMISP(misp_url, misp_key, misp_verifycert) + template = pymisp.get_object_templates_list() + if 'response' in template.keys(): + template = template['response'] try: - template_id = [x['ObjectTemplate']['id'] for x in pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == args.type][0] + template_ids = [x['ObjectTemplate']['id'] for x in template if x['ObjectTemplate']['name'] == args.type] + if len(template_ids) > 0: + template_id = template_ids[0] + else: + raise IndexError except IndexError: - valid_types = ", ".join([x['ObjectTemplate']['name'] for x in pymisp.get_object_templates_list()]) + valid_types = ", ".join([x['ObjectTemplate']['name'] for x in template]) print ("Template for type %s not found! Valid types are: %s" % (args.type, valid_types)) exit() diff --git a/examples/events/create_massive_dummy_events.py b/examples/events/create_massive_dummy_events.py index 12a2826..7829ad6 100755 --- a/examples/events/create_massive_dummy_events.py +++ b/examples/events/create_massive_dummy_events.py @@ -1,8 +1,12 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from pymisp import PyMISP -from keys import url, key +from pymisp import ExpandedPyMISP +try: + from keys import url, key +except ImportError: + url = 'http://localhost:8080' + key = '8h0gHbhS0fv6JUOlTED0AznLXFbf83TYtQrCycqb' import argparse import tools @@ -13,7 +17,7 @@ if __name__ == '__main__': parser.add_argument("-a", "--attribute", type=int, help="Number of attributes per event (default 3000)") args = parser.parse_args() - misp = PyMISP(url, key, True, 'json') + misp = ExpandedPyMISP(url, key, True) if args.limit is None: args.limit = 1 diff --git a/examples/events/tools.py b/examples/events/tools.py index 9d0e3f5..94f5d91 100644 --- a/examples/events/tools.py +++ b/examples/events/tools.py @@ -4,6 +4,7 @@ import random from random import randint import string +from pymisp import MISPEvent def randomStringGenerator(size, chars=string.ascii_lowercase + string.digits): @@ -63,12 +64,16 @@ def create_dummy_event(misp): def create_massive_dummy_events(misp, nbattribute): - event = misp.new_event(0, 4, 0, 'massive dummy event') - eventid = event['Event']['id'] + event = MISPEvent() + event.info = 'massive dummy event' + event = misp.add_event(event) + print(event) + eventid = event.id + distribution = '0' functions = [floodtxt, floodip, flooddomain, flooddomainip, floodemail, floodattachment] for i in range(nbattribute): choice = randint(0, 5) if choice == 5: - floodattachment(misp, eventid, event['Event']['distribution'], False, 'Payload delivery', '', event['Event']['info'], event['Event']['analysis'], event['Event']['threat_level_id']) + floodattachment(misp, eventid, distribution, False, 'Payload delivery', '', event.info, event.analysis, event.threat_level_id) else: functions[choice](misp, event) diff --git a/examples/get_csv.py b/examples/get_csv.py index 33baf62..5921e53 100755 --- a/examples/get_csv.py +++ b/examples/get_csv.py @@ -1,9 +1,9 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- import argparse -from pymisp import PyMISP +from pymisp import ExpandedPyMISP from keys import misp_url, misp_key, misp_verifycert @@ -14,12 +14,20 @@ if __name__ == '__main__': parser.add_argument("-o", "--object_attribute", nargs='+', help="Object attribute column names") parser.add_argument("-t", "--misp_types", nargs='+', help="MISP types to fetch (ip-src, hostname, ...)") parser.add_argument("-c", "--context", action='store_true', help="Add event level context (tags...)") - parser.add_argument("-i", "--ignore", action='store_true', help="Returns the attributes even if the event isn't published, or the attribute doesn't have the to_ids flag") parser.add_argument("-f", "--outfile", help="Output file to write the CSV.") args = parser.parse_args() - pymisp = PyMISP(misp_url, misp_key, misp_verifycert, debug=True) - response = pymisp.get_csv(args.event_id, args.attribute, args.object_attribute, args.misp_types, args.context, args.ignore) + pymisp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert, debug=True) + attr = [] + if args.attribute: + attr += args.attribute + if args.object_attribute: + attr += args.object_attribute + if not attr: + attr = None + print(args.context) + response = pymisp.search(return_format='csv', eventid=args.event_id, requested_attributes=attr, + type_attribute=args.misp_types, include_context=args.context) if args.outfile: with open(args.outfile, 'w') as f: diff --git a/pymisp/__init__.py b/pymisp/__init__.py index d613fa9..54800cf 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.4.96' +__version__ = '2.4.99' import logging import functools import warnings @@ -32,7 +32,7 @@ def deprecated(func): try: - from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate, PyMISPInvalidFormat, MISPServerError, PyMISPNotImplementedYet, PyMISPUnexpectedResponse # noqa + from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate, PyMISPInvalidFormat, MISPServerError, PyMISPNotImplementedYet, PyMISPUnexpectedResponse, PyMISPEmptyResponse # noqa from .api import PyMISP # noqa from .abstract import AbstractMISP, MISPEncode, MISPTag, Distribution, ThreatLevel, Analysis # noqa from .mispevent import MISPEvent, MISPAttribute, MISPObjectReference, MISPObjectAttribute, MISPObject, MISPUser, MISPOrganisation, MISPSighting, MISPLog # noqa diff --git a/pymisp/api.py b/pymisp/api.py index 014a13a..57e0b04 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -15,7 +15,7 @@ from io import BytesIO, open import zipfile from . import __version__, deprecated -from .exceptions import PyMISPError, SearchError, NoURL, NoKey +from .exceptions import PyMISPError, SearchError, NoURL, NoKey, PyMISPEmptyResponse from .mispevent import MISPEvent, MISPAttribute, MISPUser, MISPOrganisation, MISPSighting, MISPFeed, MISPObject from .abstract import AbstractMISP, MISPEncode @@ -157,6 +157,11 @@ class PyMISP(object): if data is None: req = requests.Request(request_type, url) else: + if not isinstance(data, str): + if isinstance(data, dict): + # Remove None values. + data = {k: v for k, v in data.items() if v is not None} + data = json.dumps(data) req = requests.Request(request_type, url, data=data) if self.asynch and background_callback is not None: local_session = FuturesSession @@ -227,6 +232,8 @@ class PyMISP(object): json_response = response.json() except ValueError: # If the server didn't return a JSON blob, we've a problem. + if not len(response.text): + raise PyMISPEmptyResponse('The server returned an empty response. \n{}\n{}\n'.format(response.request.headers, response.request.body)) raise PyMISPError(everything_broken.format(response.request.headers, response.request.body, response.text)) errors = [] @@ -336,6 +343,24 @@ class PyMISP(object): response = self._prepare_request('GET', url) return self._check_response(response) + def get_object(self, obj_id): + """Get an object + + :param obj_id: Object id to get + """ + url = urljoin(self.root_url, 'objects/view/{}'.format(obj_id)) + response = self._prepare_request('GET', url) + return self._check_response(response) + + def get_attribute(self, att_id): + """Get an attribute + + :param att_id: Attribute id to get + """ + url = urljoin(self.root_url, 'attributes/view/{}'.format(att_id)) + response = self._prepare_request('GET', url) + return self._check_response(response) + def add_event(self, event): """Add a new event @@ -1212,12 +1237,14 @@ class PyMISP(object): query['event_timestamp'] = kwargs.pop('event_timestamp', None) query['includeProposals'] = kwargs.pop('includeProposals', None) + if kwargs: + logger.info('Some unknown parameters are in kwargs. appending as-is: {}'.format(', '.join(kwargs.keys()))) + # Add all other keys as-is. + query.update({k: v for k, v in kwargs.items()}) + # Cleanup query = {k: v for k, v in query.items() if v is not None} - if kwargs: - raise SearchError('Unused parameter: {}'.format(', '.join(kwargs.keys()))) - # Create a session, make it async if and only if we have a callback return self.__query('restSearch/download', query, controller, async_callback) @@ -1410,6 +1437,15 @@ class PyMISP(object): response = self._prepare_request('GET', url) return self._check_response(response) + def get_users_statistics(self, context='data'): + """Get users statistics from the MISP instance""" + availables_contexts = ['data', 'orgs', 'users', 'tags', 'attributehistogram', 'sightings', 'attackMatrix'] + if context not in availables_contexts: + context = 'data' + url = urljoin(self.root_url, 'users/statistics/{}.json'.format(context)) + response = self._prepare_request('GET', url) + return self._check_response(response) + # ############## Sightings ################## def sighting_per_id(self, attribute_id): @@ -2270,11 +2306,9 @@ class PyMISP(object): def get_object_template_id(self, object_uuid): """Gets the template ID corresponting the UUID passed as parameter""" - templates = self.get_object_templates_list() - for t in templates: - if t['ObjectTemplate']['uuid'] == object_uuid: - return t['ObjectTemplate']['id'] - raise Exception('Unable to find template uuid {} on the MISP instance'.format(object_uuid)) + url = urljoin(self.root_url, 'objectTemplates/view/{}'.format(object_uuid)) + response = self._prepare_request('GET', url) + return self._check_response(response) def update_object_templates(self): url = urljoin(self.root_url, '/objectTemplates/update') diff --git a/pymisp/aping.py b/pymisp/aping.py index dfd28f7..097b8f9 100644 --- a/pymisp/aping.py +++ b/pymisp/aping.py @@ -6,7 +6,6 @@ from .api import PyMISP, everything_broken from .mispevent import MISPEvent, MISPAttribute, MISPSighting, MISPLog from typing import TypeVar, Optional, Tuple, List, Dict from datetime import date, datetime -import json import csv import logging @@ -70,7 +69,7 @@ class ExpandedPyMISP(PyMISP): # The server returns a json message with the error details error_message = response.json() logger.error(f'Something went wrong ({response.status_code}): {error_message}') - return {'errors': [(response.status_code, error_message)]} + return {'errors': (response.status_code, error_message)} # At this point, we had no error. @@ -80,11 +79,15 @@ class ExpandedPyMISP(PyMISP): logger.debug(response) if isinstance(response, dict) and response.get('response') is not None: # Cleanup. - return response.get('response') + response = response['response'] return response except Exception: if logger.isEnabledFor(logging.DEBUG): logger.debug(response.text) + if not len(response.content): + # Empty response + logger.error('Got an empty response.') + return {'errors': 'The response is empty.'} return response.text def get_event(self, event_id: int): @@ -97,6 +100,8 @@ class ExpandedPyMISP(PyMISP): created_event = super().add_event(event) if isinstance(created_event, str): raise NewEventError(f'Unexpected response from server: {created_event}') + elif 'errors' in created_event: + return created_event e = MISPEvent() e.load(created_event) return e @@ -105,6 +110,8 @@ class ExpandedPyMISP(PyMISP): updated_event = super().update_event(event.uuid, event) if isinstance(updated_event, str): raise UpdateEventError(f'Unexpected response from server: {updated_event}') + elif 'errors' in updated_event: + return updated_event e = MISPEvent() e.load(updated_event) return e @@ -113,6 +120,8 @@ class ExpandedPyMISP(PyMISP): updated_attribute = super().update_attribute(attribute.uuid, attribute) if isinstance(updated_attribute, str): raise UpdateAttributeError(f'Unexpected response from server: {updated_attribute}') + elif 'errors' in updated_attribute: + return updated_attribute a = MISPAttribute() a.from_dict(**updated_attribute) return a @@ -172,10 +181,7 @@ class ExpandedPyMISP(PyMISP): query['includeEvent'] = include_event_meta url = urljoin(self.root_url, url_path) - # Remove None values. - # TODO: put that in self._prepare_request - query = {k: v for k, v in query.items() if v is not None} - response = self._prepare_request('POST', url, data=json.dumps(query)) + response = self._prepare_request('POST', url, data=query) normalized_response = self._check_response(response) if isinstance(normalized_response, str) or (isinstance(normalized_response, dict) and normalized_response.get('errors')): @@ -348,10 +354,7 @@ class ExpandedPyMISP(PyMISP): query['includeContext'] = include_context query['headerless'] = headerless url = urljoin(self.root_url, f'{controller}/restSearch') - # Remove None values. - # TODO: put that in self._prepare_request - query = {k: v for k, v in query.items() if v is not None} - response = self._prepare_request('POST', url, data=json.dumps(query)) + response = self._prepare_request('POST', url, data=query) normalized_response = self._check_response(response) if return_format == 'csv' and pythonify and not headerless: return self._csv_to_dict(normalized_response) @@ -420,10 +423,7 @@ class ExpandedPyMISP(PyMISP): query['id'] = query.pop('log_id') url = urljoin(self.root_url, 'admin/logs/index') - # Remove None values. - # TODO: put that in self._prepare_request - query = {k: v for k, v in query.items() if v is not None} - response = self._prepare_request('POST', url, data=json.dumps(query)) + response = self._prepare_request('POST', url, data=query) normalized_response = self._check_response(response) if not pythonify: return normalized_response @@ -477,10 +477,7 @@ class ExpandedPyMISP(PyMISP): query['timestamp'] = self.make_timestamp(timestamp) url = urljoin(self.root_url, 'events/index') - # Remove None values. - # TODO: put that in self._prepare_request - query = {k: v for k, v in query.items() if v is not None} - response = self._prepare_request('POST', url, data=json.dumps(query)) + response = self._prepare_request('POST', url, data=query) normalized_response = self._check_response(response) if not pythonify: diff --git a/pymisp/data/describeTypes.json b/pymisp/data/describeTypes.json index 497c44b..015b943 100644 --- a/pymisp/data/describeTypes.json +++ b/pymisp/data/describeTypes.json @@ -1,24 +1,824 @@ { "result": { + "sane_defaults": { + "md5": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "sha1": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "sha256": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "pdb": { + "default_category": "Artifacts dropped", + "to_ids": 0 + }, + "filename|md5": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha1": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha256": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "ip-src": { + "default_category": "Network activity", + "to_ids": 1 + }, + "ip-dst": { + "default_category": "Network activity", + "to_ids": 1 + }, + "hostname": { + "default_category": "Network activity", + "to_ids": 1 + }, + "domain": { + "default_category": "Network activity", + "to_ids": 1 + }, + "domain|ip": { + "default_category": "Network activity", + "to_ids": 1 + }, + "email-src": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "email-dst": { + "default_category": "Network activity", + "to_ids": 1 + }, + "email-subject": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "email-attachment": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "email-body": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "float": { + "default_category": "Other", + "to_ids": 0 + }, + "url": { + "default_category": "Network activity", + "to_ids": 1 + }, + "http-method": { + "default_category": "Network activity", + "to_ids": 0 + }, + "user-agent": { + "default_category": "Network activity", + "to_ids": 0 + }, + "ja3-fingerprint-md5": { + "default_category": "Network activity", + "to_ids": 1 + }, + "hassh-md5": { + "default_category": "Network activity", + "to_ids": 1 + }, + "hasshserver-md5": { + "default_category": "Network activity", + "to_ids": 1 + }, + "regkey": { + "default_category": "Persistence mechanism", + "to_ids": 1 + }, + "regkey|value": { + "default_category": "Persistence mechanism", + "to_ids": 1 + }, + "AS": { + "default_category": "Network activity", + "to_ids": 0 + }, + "snort": { + "default_category": "Network activity", + "to_ids": 1 + }, + "bro": { + "default_category": "Network activity", + "to_ids": 1 + }, + "pattern-in-file": { + "default_category": "Payload installation", + "to_ids": 1 + }, + "pattern-in-traffic": { + "default_category": "Network activity", + "to_ids": 1 + }, + "pattern-in-memory": { + "default_category": "Payload installation", + "to_ids": 1 + }, + "yara": { + "default_category": "Payload installation", + "to_ids": 1 + }, + "stix2-pattern": { + "default_category": "Payload installation", + "to_ids": 1 + }, + "sigma": { + "default_category": "Payload installation", + "to_ids": 1 + }, + "gene": { + "default_category": "Artifacts dropped", + "to_ids": 0 + }, + "mime-type": { + "default_category": "Artifacts dropped", + "to_ids": 0 + }, + "identity-card-number": { + "default_category": "Person", + "to_ids": 0 + }, + "cookie": { + "default_category": "Network activity", + "to_ids": 0 + }, + "vulnerability": { + "default_category": "External analysis", + "to_ids": 0 + }, + "attachment": { + "default_category": "External analysis", + "to_ids": 0 + }, + "malware-sample": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "link": { + "default_category": "External analysis", + "to_ids": 0 + }, + "comment": { + "default_category": "Other", + "to_ids": 0 + }, + "text": { + "default_category": "Other", + "to_ids": 0 + }, + "hex": { + "default_category": "Other", + "to_ids": 0 + }, + "other": { + "default_category": "Other", + "to_ids": 0 + }, + "named pipe": { + "default_category": "Artifacts dropped", + "to_ids": 0 + }, + "mutex": { + "default_category": "Artifacts dropped", + "to_ids": 1 + }, + "target-user": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "target-email": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "target-machine": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "target-org": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "target-location": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "target-external": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "btc": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "xmr": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "iban": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "bic": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "bank-account-nr": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "aba-rtn": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "bin": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "cc-number": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "prtn": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "phone-number": { + "default_category": "Person", + "to_ids": 0 + }, + "threat-actor": { + "default_category": "Attribution", + "to_ids": 0 + }, + "campaign-name": { + "default_category": "Attribution", + "to_ids": 0 + }, + "campaign-id": { + "default_category": "Attribution", + "to_ids": 0 + }, + "malware-type": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "uri": { + "default_category": "Network activity", + "to_ids": 1 + }, + "authentihash": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "ssdeep": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "imphash": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "pehash": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "impfuzzy": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "sha224": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "sha384": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "sha512": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "sha512/224": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "sha512/256": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "tlsh": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "cdhash": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|authentihash": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|ssdeep": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|imphash": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|impfuzzy": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|pehash": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha224": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha384": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha512": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha512/224": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha512/256": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|tlsh": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "windows-scheduled-task": { + "default_category": "Artifacts dropped", + "to_ids": 0 + }, + "windows-service-name": { + "default_category": "Artifacts dropped", + "to_ids": 0 + }, + "windows-service-displayname": { + "default_category": "Artifacts dropped", + "to_ids": 0 + }, + "whois-registrant-email": { + "default_category": "Attribution", + "to_ids": 0 + }, + "whois-registrant-phone": { + "default_category": "Attribution", + "to_ids": 0 + }, + "whois-registrant-name": { + "default_category": "Attribution", + "to_ids": 0 + }, + "whois-registrant-org": { + "default_category": "Attribution", + "to_ids": 0 + }, + "whois-registrar": { + "default_category": "Attribution", + "to_ids": 0 + }, + "whois-creation-date": { + "default_category": "Attribution", + "to_ids": 0 + }, + "x509-fingerprint-sha1": { + "default_category": "Network activity", + "to_ids": 1 + }, + "x509-fingerprint-md5": { + "default_category": "Network activity", + "to_ids": 1 + }, + "x509-fingerprint-sha256": { + "default_category": "Network activity", + "to_ids": 1 + }, + "dns-soa-email": { + "default_category": "Attribution", + "to_ids": 0 + }, + "size-in-bytes": { + "default_category": "Other", + "to_ids": 0 + }, + "counter": { + "default_category": "Other", + "to_ids": 0 + }, + "datetime": { + "default_category": "Other", + "to_ids": 0 + }, + "cpe": { + "default_category": "Other", + "to_ids": 0 + }, + "port": { + "default_category": "Network activity", + "to_ids": 0 + }, + "ip-dst|port": { + "default_category": "Network activity", + "to_ids": 1 + }, + "ip-src|port": { + "default_category": "Network activity", + "to_ids": 1 + }, + "hostname|port": { + "default_category": "Network activity", + "to_ids": 1 + }, + "mac-address": { + "default_category": "Network activity", + "to_ids": 0 + }, + "mac-eui-64": { + "default_category": "Network activity", + "to_ids": 0 + }, + "email-dst-display-name": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "email-src-display-name": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "email-header": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "email-reply-to": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "email-x-mailer": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "email-mime-boundary": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "email-thread-index": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "email-message-id": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "github-username": { + "default_category": "Social network", + "to_ids": 0 + }, + "github-repository": { + "default_category": "Social network", + "to_ids": 0 + }, + "github-organisation": { + "default_category": "Social network", + "to_ids": 0 + }, + "jabber-id": { + "default_category": "Social network", + "to_ids": 0 + }, + "twitter-id": { + "default_category": "Social network", + "to_ids": 0 + }, + "first-name": { + "default_category": "Person", + "to_ids": 0 + }, + "middle-name": { + "default_category": "Person", + "to_ids": 0 + }, + "last-name": { + "default_category": "Person", + "to_ids": 0 + }, + "date-of-birth": { + "default_category": "Person", + "to_ids": 0 + }, + "place-of-birth": { + "default_category": "Person", + "to_ids": 0 + }, + "gender": { + "default_category": "Person", + "to_ids": 0 + }, + "passport-number": { + "default_category": "Person", + "to_ids": 0 + }, + "passport-country": { + "default_category": "Person", + "to_ids": 0 + }, + "passport-expiration": { + "default_category": "Person", + "to_ids": 0 + }, + "redress-number": { + "default_category": "Person", + "to_ids": 0 + }, + "nationality": { + "default_category": "Person", + "to_ids": 0 + }, + "visa-number": { + "default_category": "Person", + "to_ids": 0 + }, + "issue-date-of-the-visa": { + "default_category": "Person", + "to_ids": 0 + }, + "primary-residence": { + "default_category": "Person", + "to_ids": 0 + }, + "country-of-residence": { + "default_category": "Person", + "to_ids": 0 + }, + "special-service-request": { + "default_category": "Person", + "to_ids": 0 + }, + "frequent-flyer-number": { + "default_category": "Person", + "to_ids": 0 + }, + "travel-details": { + "default_category": "Person", + "to_ids": 0 + }, + "payment-details": { + "default_category": "Person", + "to_ids": 0 + }, + "place-port-of-original-embarkation": { + "default_category": "Person", + "to_ids": 0 + }, + "place-port-of-clearance": { + "default_category": "Person", + "to_ids": 0 + }, + "place-port-of-onward-foreign-destination": { + "default_category": "Person", + "to_ids": 0 + }, + "passenger-name-record-locator-number": { + "default_category": "Person", + "to_ids": 0 + }, + "mobile-application-id": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "cortex": { + "default_category": "External analysis", + "to_ids": 0 + }, + "boolean": { + "default_category": "Other", + "to_ids": 0 + } + }, + "types": [ + "md5", + "sha1", + "sha256", + "filename", + "pdb", + "filename|md5", + "filename|sha1", + "filename|sha256", + "ip-src", + "ip-dst", + "hostname", + "domain", + "domain|ip", + "email-src", + "email-dst", + "email-subject", + "email-attachment", + "email-body", + "float", + "url", + "http-method", + "user-agent", + "ja3-fingerprint-md5", + "hassh-md5", + "hasshserver-md5", + "regkey", + "regkey|value", + "AS", + "snort", + "bro", + "pattern-in-file", + "pattern-in-traffic", + "pattern-in-memory", + "yara", + "stix2-pattern", + "sigma", + "gene", + "mime-type", + "identity-card-number", + "cookie", + "vulnerability", + "attachment", + "malware-sample", + "link", + "comment", + "text", + "hex", + "other", + "named pipe", + "mutex", + "target-user", + "target-email", + "target-machine", + "target-org", + "target-location", + "target-external", + "btc", + "xmr", + "iban", + "bic", + "bank-account-nr", + "aba-rtn", + "bin", + "cc-number", + "prtn", + "phone-number", + "threat-actor", + "campaign-name", + "campaign-id", + "malware-type", + "uri", + "authentihash", + "ssdeep", + "imphash", + "pehash", + "impfuzzy", + "sha224", + "sha384", + "sha512", + "sha512/224", + "sha512/256", + "tlsh", + "cdhash", + "filename|authentihash", + "filename|ssdeep", + "filename|imphash", + "filename|impfuzzy", + "filename|pehash", + "filename|sha224", + "filename|sha384", + "filename|sha512", + "filename|sha512/224", + "filename|sha512/256", + "filename|tlsh", + "windows-scheduled-task", + "windows-service-name", + "windows-service-displayname", + "whois-registrant-email", + "whois-registrant-phone", + "whois-registrant-name", + "whois-registrant-org", + "whois-registrar", + "whois-creation-date", + "x509-fingerprint-sha1", + "x509-fingerprint-md5", + "x509-fingerprint-sha256", + "dns-soa-email", + "size-in-bytes", + "counter", + "datetime", + "cpe", + "port", + "ip-dst|port", + "ip-src|port", + "hostname|port", + "mac-address", + "mac-eui-64", + "email-dst-display-name", + "email-src-display-name", + "email-header", + "email-reply-to", + "email-x-mailer", + "email-mime-boundary", + "email-thread-index", + "email-message-id", + "github-username", + "github-repository", + "github-organisation", + "jabber-id", + "twitter-id", + "first-name", + "middle-name", + "last-name", + "date-of-birth", + "place-of-birth", + "gender", + "passport-number", + "passport-country", + "passport-expiration", + "redress-number", + "nationality", + "visa-number", + "issue-date-of-the-visa", + "primary-residence", + "country-of-residence", + "special-service-request", + "frequent-flyer-number", + "travel-details", + "payment-details", + "place-port-of-original-embarkation", + "place-port-of-clearance", + "place-port-of-onward-foreign-destination", + "passenger-name-record-locator-number", + "mobile-application-id", + "cortex", + "boolean" + ], "categories": [ + "Internal reference", + "Targeting data", "Antivirus detection", + "Payload delivery", "Artifacts dropped", + "Payload installation", + "Persistence mechanism", + "Network activity", + "Payload type", "Attribution", "External analysis", "Financial fraud", - "Internal reference", - "Network activity", - "Other", - "Payload delivery", - "Payload installation", - "Payload type", - "Persistence mechanism", - "Person", - "Social network", "Support Tool", - "Targeting data" + "Social network", + "Person", + "Other" ], "category_type_mappings": { + "Internal reference": [ + "text", + "link", + "comment", + "other", + "hex" + ], + "Targeting data": [ + "target-user", + "target-email", + "target-machine", + "target-org", + "target-location", + "target-external", + "comment" + ], "Antivirus detection": [ "link", "comment", @@ -27,6 +827,86 @@ "attachment", "other" ], + "Payload delivery": [ + "md5", + "sha1", + "sha224", + "sha256", + "sha384", + "sha512", + "sha512/224", + "sha512/256", + "ssdeep", + "imphash", + "impfuzzy", + "authentihash", + "pehash", + "tlsh", + "cdhash", + "filename", + "filename|md5", + "filename|sha1", + "filename|sha224", + "filename|sha256", + "filename|sha384", + "filename|sha512", + "filename|sha512/224", + "filename|sha512/256", + "filename|authentihash", + "filename|ssdeep", + "filename|tlsh", + "filename|imphash", + "filename|impfuzzy", + "filename|pehash", + "mac-address", + "mac-eui-64", + "ip-src", + "ip-dst", + "ip-dst|port", + "ip-src|port", + "hostname", + "domain", + "email-src", + "email-dst", + "email-subject", + "email-attachment", + "email-body", + "url", + "user-agent", + "AS", + "pattern-in-file", + "pattern-in-traffic", + "stix2-pattern", + "yara", + "sigma", + "mime-type", + "attachment", + "malware-sample", + "link", + "malware-type", + "comment", + "text", + "hex", + "vulnerability", + "x509-fingerprint-sha1", + "x509-fingerprint-md5", + "x509-fingerprint-sha256", + "ja3-fingerprint-md5", + "hassh-md5", + "hasshserver-md5", + "other", + "hostname|port", + "email-dst-display-name", + "email-src-display-name", + "email-header", + "email-reply-to", + "email-x-mailer", + "email-mime-boundary", + "email-thread-index", + "email-message-id", + "mobile-application-id", + "whois-registrant-email" + ], "Artifacts dropped": [ "md5", "sha1", @@ -40,6 +920,7 @@ "imphash", "impfuzzy", "authentihash", + "cdhash", "filename", "filename|md5", "filename|sha1", @@ -81,6 +962,107 @@ "gene", "mime-type" ], + "Payload installation": [ + "md5", + "sha1", + "sha224", + "sha256", + "sha384", + "sha512", + "sha512/224", + "sha512/256", + "ssdeep", + "imphash", + "impfuzzy", + "authentihash", + "pehash", + "tlsh", + "cdhash", + "filename", + "filename|md5", + "filename|sha1", + "filename|sha224", + "filename|sha256", + "filename|sha384", + "filename|sha512", + "filename|sha512/224", + "filename|sha512/256", + "filename|authentihash", + "filename|ssdeep", + "filename|tlsh", + "filename|imphash", + "filename|impfuzzy", + "filename|pehash", + "pattern-in-file", + "pattern-in-traffic", + "pattern-in-memory", + "stix2-pattern", + "yara", + "sigma", + "vulnerability", + "attachment", + "malware-sample", + "malware-type", + "comment", + "text", + "hex", + "x509-fingerprint-sha1", + "x509-fingerprint-md5", + "x509-fingerprint-sha256", + "mobile-application-id", + "other", + "mime-type" + ], + "Persistence mechanism": [ + "filename", + "regkey", + "regkey|value", + "comment", + "text", + "other", + "hex" + ], + "Network activity": [ + "ip-src", + "ip-dst", + "ip-dst|port", + "ip-src|port", + "port", + "hostname", + "domain", + "domain|ip", + "mac-address", + "mac-eui-64", + "email-dst", + "url", + "uri", + "user-agent", + "http-method", + "AS", + "snort", + "pattern-in-file", + "stix2-pattern", + "pattern-in-traffic", + "attachment", + "comment", + "text", + "x509-fingerprint-md5", + "x509-fingerprint-sha1", + "x509-fingerprint-sha256", + "ja3-fingerprint-md5", + "hassh-md5", + "hasshserver-md5", + "other", + "hex", + "cookie", + "hostname|port", + "bro" + ], + "Payload type": [ + "comment", + "text", + "other" + ], "Attribution": [ "threat-actor", "campaign-name", @@ -135,6 +1117,9 @@ "x509-fingerprint-sha1", "x509-fingerprint-md5", "x509-fingerprint-sha256", + "ja3-fingerprint-md5", + "hassh-md5", + "hasshserver-md5", "github-repository", "other", "cortex" @@ -155,198 +1140,27 @@ "other", "hex" ], - "Internal reference": [ - "text", + "Support Tool": [ "link", + "text", + "attachment", "comment", "other", "hex" ], - "Network activity": [ - "ip-src", - "ip-dst", - "ip-dst|port", - "ip-src|port", - "port", - "hostname", - "domain", - "domain|ip", - "mac-address", - "mac-eui-64", - "email-dst", - "url", - "uri", - "user-agent", - "http-method", - "AS", - "snort", - "pattern-in-file", - "stix2-pattern", - "pattern-in-traffic", - "attachment", - "comment", - "text", - "x509-fingerprint-sha1", - "other", - "hex", - "cookie", - "hostname|port", - "bro" - ], - "Other": [ - "comment", - "text", - "other", - "size-in-bytes", - "counter", - "datetime", - "cpe", - "port", - "float", - "hex", - "phone-number", - "boolean" - ], - "Payload delivery": [ - "md5", - "sha1", - "sha224", - "sha256", - "sha384", - "sha512", - "sha512/224", - "sha512/256", - "ssdeep", - "imphash", - "impfuzzy", - "authentihash", - "pehash", - "tlsh", - "filename", - "filename|md5", - "filename|sha1", - "filename|sha224", - "filename|sha256", - "filename|sha384", - "filename|sha512", - "filename|sha512/224", - "filename|sha512/256", - "filename|authentihash", - "filename|ssdeep", - "filename|tlsh", - "filename|imphash", - "filename|impfuzzy", - "filename|pehash", - "mac-address", - "mac-eui-64", - "ip-src", - "ip-dst", - "ip-dst|port", - "ip-src|port", - "hostname", - "domain", + "Social network": [ + "github-username", + "github-repository", + "github-organisation", + "jabber-id", + "twitter-id", "email-src", "email-dst", - "email-subject", - "email-attachment", - "email-body", - "url", - "user-agent", - "AS", - "pattern-in-file", - "pattern-in-traffic", - "stix2-pattern", - "yara", - "sigma", - "mime-type", - "attachment", - "malware-sample", - "link", - "malware-type", "comment", "text", - "hex", - "vulnerability", - "x509-fingerprint-sha1", - "x509-fingerprint-md5", - "x509-fingerprint-sha256", "other", - "hostname|port", - "email-dst-display-name", - "email-src-display-name", - "email-header", - "email-reply-to", - "email-x-mailer", - "email-mime-boundary", - "email-thread-index", - "email-message-id", - "mobile-application-id", "whois-registrant-email" ], - "Payload installation": [ - "md5", - "sha1", - "sha224", - "sha256", - "sha384", - "sha512", - "sha512/224", - "sha512/256", - "ssdeep", - "imphash", - "impfuzzy", - "authentihash", - "pehash", - "tlsh", - "filename", - "filename|md5", - "filename|sha1", - "filename|sha224", - "filename|sha256", - "filename|sha384", - "filename|sha512", - "filename|sha512/224", - "filename|sha512/256", - "filename|authentihash", - "filename|ssdeep", - "filename|tlsh", - "filename|imphash", - "filename|impfuzzy", - "filename|pehash", - "pattern-in-file", - "pattern-in-traffic", - "pattern-in-memory", - "stix2-pattern", - "yara", - "sigma", - "vulnerability", - "attachment", - "malware-sample", - "malware-type", - "comment", - "text", - "hex", - "x509-fingerprint-sha1", - "x509-fingerprint-md5", - "x509-fingerprint-sha256", - "mobile-application-id", - "other", - "mime-type" - ], - "Payload type": [ - "comment", - "text", - "other" - ], - "Persistence mechanism": [ - "filename", - "regkey", - "regkey|value", - "comment", - "text", - "other", - "hex" - ], "Person": [ "first-name", "middle-name", @@ -377,800 +1191,20 @@ "phone-number", "identity-card-number" ], - "Social network": [ - "github-username", - "github-repository", - "github-organisation", - "jabber-id", - "twitter-id", - "email-src", - "email-dst", + "Other": [ "comment", "text", "other", - "whois-registrant-email" - ], - "Support Tool": [ - "link", - "text", - "attachment", - "comment", - "other", - "hex" - ], - "Targeting data": [ - "target-user", - "target-email", - "target-machine", - "target-org", - "target-location", - "target-external", - "comment" + "size-in-bytes", + "counter", + "datetime", + "cpe", + "port", + "float", + "hex", + "phone-number", + "boolean" ] - }, - "sane_defaults": { - "AS": { - "default_category": "Network activity", - "to_ids": 0 - }, - "aba-rtn": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "attachment": { - "default_category": "External analysis", - "to_ids": 0 - }, - "authentihash": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "bank-account-nr": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "bic": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "bin": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "boolean": { - "default_category": "Other", - "to_ids": 0 - }, - "bro": { - "default_category": "Network activity", - "to_ids": 1 - }, - "btc": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "campaign-id": { - "default_category": "Attribution", - "to_ids": 0 - }, - "campaign-name": { - "default_category": "Attribution", - "to_ids": 0 - }, - "cc-number": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "comment": { - "default_category": "Other", - "to_ids": 0 - }, - "cookie": { - "default_category": "Network activity", - "to_ids": 0 - }, - "cortex": { - "default_category": "External analysis", - "to_ids": 0 - }, - "counter": { - "default_category": "Other", - "to_ids": 0 - }, - "country-of-residence": { - "default_category": "Person", - "to_ids": 0 - }, - "cpe": { - "default_category": "Other", - "to_ids": 0 - }, - "date-of-birth": { - "default_category": "Person", - "to_ids": 0 - }, - "datetime": { - "default_category": "Other", - "to_ids": 0 - }, - "dns-soa-email": { - "default_category": "Attribution", - "to_ids": 0 - }, - "domain": { - "default_category": "Network activity", - "to_ids": 1 - }, - "domain|ip": { - "default_category": "Network activity", - "to_ids": 1 - }, - "email-attachment": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "email-body": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-dst": { - "default_category": "Network activity", - "to_ids": 1 - }, - "email-dst-display-name": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-header": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-message-id": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-mime-boundary": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-reply-to": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-src": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "email-src-display-name": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-subject": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-thread-index": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-x-mailer": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "filename": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|authentihash": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|impfuzzy": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|imphash": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|md5": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|pehash": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|sha1": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|sha224": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|sha256": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|sha384": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|sha512": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|sha512/224": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|sha512/256": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|ssdeep": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|tlsh": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "first-name": { - "default_category": "Person", - "to_ids": 0 - }, - "float": { - "default_category": "Other", - "to_ids": 0 - }, - "frequent-flyer-number": { - "default_category": "Person", - "to_ids": 0 - }, - "gender": { - "default_category": "Person", - "to_ids": 0 - }, - "gene": { - "default_category": "Artifacts dropped", - "to_ids": 0 - }, - "github-organisation": { - "default_category": "Social network", - "to_ids": 0 - }, - "github-repository": { - "default_category": "Social network", - "to_ids": 0 - }, - "github-username": { - "default_category": "Social network", - "to_ids": 0 - }, - "hex": { - "default_category": "Other", - "to_ids": 0 - }, - "hostname": { - "default_category": "Network activity", - "to_ids": 1 - }, - "hostname|port": { - "default_category": "Network activity", - "to_ids": 1 - }, - "http-method": { - "default_category": "Network activity", - "to_ids": 0 - }, - "iban": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "identity-card-number": { - "default_category": "Person", - "to_ids": 0 - }, - "impfuzzy": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "imphash": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "ip-dst": { - "default_category": "Network activity", - "to_ids": 1 - }, - "ip-dst|port": { - "default_category": "Network activity", - "to_ids": 1 - }, - "ip-src": { - "default_category": "Network activity", - "to_ids": 1 - }, - "ip-src|port": { - "default_category": "Network activity", - "to_ids": 1 - }, - "issue-date-of-the-visa": { - "default_category": "Person", - "to_ids": 0 - }, - "jabber-id": { - "default_category": "Social network", - "to_ids": 0 - }, - "last-name": { - "default_category": "Person", - "to_ids": 0 - }, - "link": { - "default_category": "External analysis", - "to_ids": 0 - }, - "mac-address": { - "default_category": "Network activity", - "to_ids": 0 - }, - "mac-eui-64": { - "default_category": "Network activity", - "to_ids": 0 - }, - "malware-sample": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "malware-type": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "md5": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "middle-name": { - "default_category": "Person", - "to_ids": 0 - }, - "mime-type": { - "default_category": "Artifacts dropped", - "to_ids": 0 - }, - "mobile-application-id": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "mutex": { - "default_category": "Artifacts dropped", - "to_ids": 1 - }, - "named pipe": { - "default_category": "Artifacts dropped", - "to_ids": 0 - }, - "nationality": { - "default_category": "Person", - "to_ids": 0 - }, - "other": { - "default_category": "Other", - "to_ids": 0 - }, - "passenger-name-record-locator-number": { - "default_category": "Person", - "to_ids": 0 - }, - "passport-country": { - "default_category": "Person", - "to_ids": 0 - }, - "passport-expiration": { - "default_category": "Person", - "to_ids": 0 - }, - "passport-number": { - "default_category": "Person", - "to_ids": 0 - }, - "pattern-in-file": { - "default_category": "Payload installation", - "to_ids": 1 - }, - "pattern-in-memory": { - "default_category": "Payload installation", - "to_ids": 1 - }, - "pattern-in-traffic": { - "default_category": "Network activity", - "to_ids": 1 - }, - "payment-details": { - "default_category": "Person", - "to_ids": 0 - }, - "pdb": { - "default_category": "Artifacts dropped", - "to_ids": 0 - }, - "pehash": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "phone-number": { - "default_category": "Person", - "to_ids": 0 - }, - "place-of-birth": { - "default_category": "Person", - "to_ids": 0 - }, - "place-port-of-clearance": { - "default_category": "Person", - "to_ids": 0 - }, - "place-port-of-onward-foreign-destination": { - "default_category": "Person", - "to_ids": 0 - }, - "place-port-of-original-embarkation": { - "default_category": "Person", - "to_ids": 0 - }, - "port": { - "default_category": "Network activity", - "to_ids": 0 - }, - "primary-residence": { - "default_category": "Person", - "to_ids": 0 - }, - "prtn": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "redress-number": { - "default_category": "Person", - "to_ids": 0 - }, - "regkey": { - "default_category": "Persistence mechanism", - "to_ids": 1 - }, - "regkey|value": { - "default_category": "Persistence mechanism", - "to_ids": 1 - }, - "sha1": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "sha224": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "sha256": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "sha384": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "sha512": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "sha512/224": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "sha512/256": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "sigma": { - "default_category": "Payload installation", - "to_ids": 1 - }, - "size-in-bytes": { - "default_category": "Other", - "to_ids": 0 - }, - "snort": { - "default_category": "Network activity", - "to_ids": 1 - }, - "special-service-request": { - "default_category": "Person", - "to_ids": 0 - }, - "ssdeep": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "stix2-pattern": { - "default_category": "Payload installation", - "to_ids": 1 - }, - "target-email": { - "default_category": "Targeting data", - "to_ids": 0 - }, - "target-external": { - "default_category": "Targeting data", - "to_ids": 0 - }, - "target-location": { - "default_category": "Targeting data", - "to_ids": 0 - }, - "target-machine": { - "default_category": "Targeting data", - "to_ids": 0 - }, - "target-org": { - "default_category": "Targeting data", - "to_ids": 0 - }, - "target-user": { - "default_category": "Targeting data", - "to_ids": 0 - }, - "text": { - "default_category": "Other", - "to_ids": 0 - }, - "threat-actor": { - "default_category": "Attribution", - "to_ids": 0 - }, - "tlsh": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "travel-details": { - "default_category": "Person", - "to_ids": 0 - }, - "twitter-id": { - "default_category": "Social network", - "to_ids": 0 - }, - "uri": { - "default_category": "Network activity", - "to_ids": 1 - }, - "url": { - "default_category": "Network activity", - "to_ids": 1 - }, - "user-agent": { - "default_category": "Network activity", - "to_ids": 0 - }, - "visa-number": { - "default_category": "Person", - "to_ids": 0 - }, - "vulnerability": { - "default_category": "External analysis", - "to_ids": 0 - }, - "whois-creation-date": { - "default_category": "Attribution", - "to_ids": 0 - }, - "whois-registrant-email": { - "default_category": "Attribution", - "to_ids": 0 - }, - "whois-registrant-name": { - "default_category": "Attribution", - "to_ids": 0 - }, - "whois-registrant-org": { - "default_category": "Attribution", - "to_ids": 0 - }, - "whois-registrant-phone": { - "default_category": "Attribution", - "to_ids": 0 - }, - "whois-registrar": { - "default_category": "Attribution", - "to_ids": 0 - }, - "windows-scheduled-task": { - "default_category": "Artifacts dropped", - "to_ids": 0 - }, - "windows-service-displayname": { - "default_category": "Artifacts dropped", - "to_ids": 0 - }, - "windows-service-name": { - "default_category": "Artifacts dropped", - "to_ids": 0 - }, - "x509-fingerprint-md5": { - "default_category": "Network activity", - "to_ids": 1 - }, - "x509-fingerprint-sha1": { - "default_category": "Network activity", - "to_ids": 1 - }, - "x509-fingerprint-sha256": { - "default_category": "Network activity", - "to_ids": 1 - }, - "xmr": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "yara": { - "default_category": "Payload installation", - "to_ids": 1 - } - }, - "types": [ - "AS", - "aba-rtn", - "attachment", - "authentihash", - "bank-account-nr", - "bic", - "bin", - "boolean", - "bro", - "btc", - "campaign-id", - "campaign-name", - "cc-number", - "comment", - "cookie", - "cortex", - "counter", - "country-of-residence", - "cpe", - "date-of-birth", - "datetime", - "dns-soa-email", - "domain", - "domain|ip", - "email-attachment", - "email-body", - "email-dst", - "email-dst-display-name", - "email-header", - "email-message-id", - "email-mime-boundary", - "email-reply-to", - "email-src", - "email-src-display-name", - "email-subject", - "email-thread-index", - "email-x-mailer", - "filename", - "filename|authentihash", - "filename|impfuzzy", - "filename|imphash", - "filename|md5", - "filename|pehash", - "filename|sha1", - "filename|sha224", - "filename|sha256", - "filename|sha384", - "filename|sha512", - "filename|sha512/224", - "filename|sha512/256", - "filename|ssdeep", - "filename|tlsh", - "first-name", - "float", - "frequent-flyer-number", - "gender", - "gene", - "github-organisation", - "github-repository", - "github-username", - "hex", - "hostname", - "hostname|port", - "http-method", - "iban", - "identity-card-number", - "impfuzzy", - "imphash", - "ip-dst", - "ip-dst|port", - "ip-src", - "ip-src|port", - "issue-date-of-the-visa", - "jabber-id", - "last-name", - "link", - "mac-address", - "mac-eui-64", - "malware-sample", - "malware-type", - "md5", - "middle-name", - "mime-type", - "mobile-application-id", - "mutex", - "named pipe", - "nationality", - "other", - "passenger-name-record-locator-number", - "passport-country", - "passport-expiration", - "passport-number", - "pattern-in-file", - "pattern-in-memory", - "pattern-in-traffic", - "payment-details", - "pdb", - "pehash", - "phone-number", - "place-of-birth", - "place-port-of-clearance", - "place-port-of-onward-foreign-destination", - "place-port-of-original-embarkation", - "port", - "primary-residence", - "prtn", - "redress-number", - "regkey", - "regkey|value", - "sha1", - "sha224", - "sha256", - "sha384", - "sha512", - "sha512/224", - "sha512/256", - "sigma", - "size-in-bytes", - "snort", - "special-service-request", - "ssdeep", - "stix2-pattern", - "target-email", - "target-external", - "target-location", - "target-machine", - "target-org", - "target-user", - "text", - "threat-actor", - "tlsh", - "travel-details", - "twitter-id", - "uri", - "url", - "user-agent", - "visa-number", - "vulnerability", - "whois-creation-date", - "whois-registrant-email", - "whois-registrant-name", - "whois-registrant-org", - "whois-registrant-phone", - "whois-registrar", - "windows-scheduled-task", - "windows-service-displayname", - "windows-service-name", - "x509-fingerprint-md5", - "x509-fingerprint-sha1", - "x509-fingerprint-sha256", - "xmr", - "yara" - ] + } } -} \ No newline at end of file +} diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index 7fe77c0..b25388c 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit 7fe77c02affc0abe14cc67fe9f14400e8b72561c +Subproject commit b25388c406e2f8dfcba94fe742a3ca90360315fd diff --git a/pymisp/exceptions.py b/pymisp/exceptions.py index 481720b..1d7663f 100644 --- a/pymisp/exceptions.py +++ b/pymisp/exceptions.py @@ -67,3 +67,7 @@ class PyMISPNotImplementedYet(PyMISPError): class PyMISPUnexpectedResponse(PyMISPError): pass + + +class PyMISPEmptyResponse(PyMISPError): + pass diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 9c532a3..ff7ad33 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -464,13 +464,7 @@ class MISPEvent(AbstractMISP): event = json_event if not event: raise PyMISPError('Invalid event') - # Invalid event created by MISP up to 2.4.52 (attribute_count is none instead of '0') - if (event.get('Event') and - 'attribute_count' in event.get('Event') and - event.get('Event').get('attribute_count') is None): - event['Event']['attribute_count'] = '0' - e = event.get('Event') - self.from_dict(**e) + self.from_dict(**event) if validate: jsonschema.validate(json.loads(self.to_json()), self.__json_schema) @@ -489,17 +483,19 @@ class MISPEvent(AbstractMISP): raise NewEventError('Invalid format for the date: {} - {}'.format(date, type(date))) def from_dict(self, **kwargs): + if kwargs.get('Event'): + kwargs = kwargs.get('Event') # Required value self.info = kwargs.pop('info', None) if self.info is None: - raise NewAttributeError('The info field of the new event is required.') + raise NewEventError('The info field of the new event is required.') # Default values for a valid event to send to a MISP instance self.distribution = kwargs.pop('distribution', None) if self.distribution is not None: self.distribution = int(self.distribution) if self.distribution not in [0, 1, 2, 3, 4]: - raise NewAttributeError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 4'.format(self.distribution)) + raise NewEventError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 4'.format(self.distribution)) if kwargs.get('threat_level_id') is not None: self.threat_level_id = int(kwargs.pop('threat_level_id')) @@ -1005,6 +1001,8 @@ class MISPObject(AbstractMISP): raise PyMISPError('All the attributes have to be of type MISPObjectReference.') def from_dict(self, **kwargs): + if kwargs.get('Object'): + kwargs = kwargs.get('Object') if self._known_template: if kwargs.get('template_uuid') and kwargs['template_uuid'] != self.template_uuid: if self._strict: @@ -1017,6 +1015,12 @@ class MISPObject(AbstractMISP): else: self._known_template = False + if 'distribution' in kwargs and kwargs['distribution'] is not None: + self.distribution = kwargs.pop('distribution') + self.distribution = int(self.distribution) + if self.distribution not in [0, 1, 2, 3, 4, 5]: + raise NewAttributeError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 4, 5'.format(self.distribution)) + if kwargs.get('timestamp'): if sys.version_info >= (3, 3): self.timestamp = datetime.datetime.fromtimestamp(int(kwargs.pop('timestamp')), datetime.timezone.utc) diff --git a/tests/testlive_comprehensive.py b/tests/testlive_comprehensive.py index b33976f..a8d147b 100644 --- a/tests/testlive_comprehensive.py +++ b/tests/testlive_comprehensive.py @@ -3,22 +3,24 @@ import unittest -from pymisp import ExpandedPyMISP, MISPEvent, MISPOrganisation, MISPUser, Distribution, ThreatLevel, Analysis +from pymisp import ExpandedPyMISP, MISPEvent, MISPOrganisation, MISPUser, Distribution, ThreatLevel, Analysis, MISPObject +from pymisp.tools import make_binary_objects from datetime import datetime, timedelta, date from io import BytesIO import time +from uuid import uuid4 try: from keys import url, key + verifycert = False + travis_run = True except ImportError as e: print(e) url = 'http://localhost:8080' - key = 'BSip0zVadeFDeolkX2g7MHx8mrlr0uE04hh6CQj0' - -from uuid import uuid4 - -travis_run = True + key = '8h0gHbhS0fv6JUOlTED0AznLXFbf83TYtQrCycqb' + verifycert = False + travis_run = False class TestComprehensive(unittest.TestCase): @@ -27,7 +29,7 @@ class TestComprehensive(unittest.TestCase): def setUpClass(cls): cls.maxDiff = None # Connect as admin - cls.admin_misp_connector = ExpandedPyMISP(url, key, debug=False) + cls.admin_misp_connector = ExpandedPyMISP(url, key, verifycert, debug=False) # Creates an org org = cls.admin_misp_connector.add_organisation(name='Test Org') cls.test_org = MISPOrganisation() @@ -36,12 +38,12 @@ class TestComprehensive(unittest.TestCase): usr = cls.admin_misp_connector.add_user(email='testusr@user.local', org_id=cls.test_org.id, role_id=3) cls.test_usr = MISPUser() cls.test_usr.from_dict(**usr) - cls.user_misp_connector = ExpandedPyMISP(url, cls.test_usr.authkey) + cls.user_misp_connector = ExpandedPyMISP(url, cls.test_usr.authkey, verifycert, debug=False) # Creates a publisher pub = cls.admin_misp_connector.add_user(email='testpub@user.local', org_id=cls.test_org.id, role_id=4) cls.test_pub = MISPUser() cls.test_pub.from_dict(**pub) - cls.pub_misp_connector = ExpandedPyMISP(url, cls.test_pub.authkey) + cls.pub_misp_connector = ExpandedPyMISP(url, cls.test_pub.authkey, verifycert) @classmethod def tearDownClass(cls): @@ -439,6 +441,51 @@ class TestComprehensive(unittest.TestCase): self.admin_misp_connector.delete_event(first.id) self.admin_misp_connector.delete_event(second.id) + def test_default_distribution(self): + '''The default distributions on the VM are This community only for the events and Inherit from event for attr/obj)''' + if travis_run: + return + first = self.create_simple_event() + del first.distribution + o = first.add_object(name='file') + o.add_attribute('filename', value='foo.exe') + try: + # Event create + first = self.user_misp_connector.add_event(first) + self.assertEqual(first.distribution, Distribution.this_community_only.value) + self.assertEqual(first.attributes[0].distribution, Distribution.inherit.value) + self.assertEqual(first.objects[0].distribution, Distribution.inherit.value) + self.assertEqual(first.objects[0].attributes[0].distribution, Distribution.inherit.value) + # Event edit + first.add_attribute('ip-dst', '12.54.76.43') + o = first.add_object(name='file') + o.add_attribute('filename', value='foo2.exe') + first = self.user_misp_connector.update_event(first) + self.assertEqual(first.attributes[1].distribution, Distribution.inherit.value) + self.assertEqual(first.objects[1].distribution, Distribution.inherit.value) + self.assertEqual(first.objects[1].attributes[0].distribution, Distribution.inherit.value) + # Attribute create + attribute = self.user_misp_connector.add_named_attribute(first, 'comment', 'bar') + # FIXME: Add helper that returns a list of MISPAttribute + self.assertEqual(attribute[0]['Attribute']['distribution'], str(Distribution.inherit.value)) + # Object - add + o = MISPObject('file') + o.add_attribute('filename', value='blah.exe') + new_obj = self.user_misp_connector.add_object(first.id, o.template_uuid, o) + # FIXME: Add helper that returns a MISPObject + self.assertEqual(new_obj['Object']['distribution'], str(Distribution.inherit.value)) + self.assertEqual(new_obj['Object']['Attribute'][0]['distribution'], str(Distribution.inherit.value)) + # Object - edit + clean_obj = MISPObject(**new_obj['Object']) + clean_obj.from_dict(**new_obj['Object']) + clean_obj.add_attribute('filename', value='blah.exe') + new_obj = self.user_misp_connector.edit_object(clean_obj) + for a in new_obj['Object']['Attribute']: + self.assertEqual(a['distribution'], str(Distribution.inherit.value)) + finally: + # Delete event + self.admin_misp_connector.delete_event(first.id) + def test_simple_event(self): '''Search a bunch of parameters: * Value not existing @@ -517,7 +564,7 @@ class TestComprehensive(unittest.TestCase): # quickfilter events = self.user_misp_connector.search(timestamp=timeframe, - quickfilter='%bar%', pythonify=True) + quickfilter='%foo blah%', pythonify=True) # FIXME: should return one event # print(events) # self.assertEqual(len(events), 1) @@ -709,7 +756,7 @@ class TestComprehensive(unittest.TestCase): second = self.user_misp_connector.add_event(second) response = self.user_misp_connector.fast_publish(first.id, alert=False) - self.assertEqual(response['errors'][0][1]['message'], 'You do not have permission to use this functionality.') + self.assertEqual(response['errors'][1]['message'], 'You do not have permission to use this functionality.') # Default search, attribute with to_ids == True first.attributes[0].to_ids = True @@ -835,6 +882,24 @@ class TestComprehensive(unittest.TestCase): self.admin_misp_connector.enable_tag(tag['id']) # FIXME: returns the tag with ID 1 + def test_add_event_with_attachment(self): + first = self.create_simple_event() + try: + first = self.user_misp_connector.add_event(first) + file_obj, bin_obj, sections = make_binary_objects('tests/viper-test-files/test_files/whoami.exe', standalone=False) + first.add_object(file_obj) + first.add_object(bin_obj) + for s in sections: + first.add_object(s) + self.assertEqual(len(first.objects[0].references), 1) + self.assertEqual(first.objects[0].references[0].relationship_type, 'included-in') + first = self.user_misp_connector.update_event(first) + self.assertEqual(len(first.objects[0].references), 1) + self.assertEqual(first.objects[0].references[0].relationship_type, 'included-in') + finally: + # Delete event + self.admin_misp_connector.delete_event(first.id) + def test_taxonomies(self): # Make sure we're up-to-date self.admin_misp_connector.update_taxonomies()