From 06b829e7813ce6d967b9b0a24385dde19e682ad2 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Mon, 13 Aug 2018 12:10:15 +0200 Subject: [PATCH 01/38] new: [usersStats] Possibility to fetch users/statistics data for all context (usage, org, tags, ...) --- pymisp/api.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pymisp/api.py b/pymisp/api.py index e189c15..cee2caf 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1366,6 +1366,15 @@ class PyMISP(object): response = self._prepare_request('GET', url) return self._check_response(response) + def get_users_statistics(self, context='data'): + """Get users statistics from the MISP instance""" + availables_contexts = ['data', 'orgs', 'users', 'tags', 'attributehistogram', 'sightings', 'attackMatrix'] + if context not in availables_contexts: + context = 'data' + url = urljoin(self.root_url, 'users/statistics/{}.json'.format(context)) + response = self._prepare_request('GET', url) + return self._check_response(response) + # ############## Sightings ################## def sighting_per_id(self, attribute_id): From de118795ceba965d588cbbf1d8bf36363e470929 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 28 Nov 2018 17:34:38 +0100 Subject: [PATCH 02/38] fix: properly handle errors on event creation/update --- pymisp/aping.py | 6 ++++++ pymisp/mispevent.py | 3 +-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pymisp/aping.py b/pymisp/aping.py index dfd28f7..9f59273 100644 --- a/pymisp/aping.py +++ b/pymisp/aping.py @@ -97,6 +97,8 @@ class ExpandedPyMISP(PyMISP): created_event = super().add_event(event) if isinstance(created_event, str): raise NewEventError(f'Unexpected response from server: {created_event}') + elif 'errors' in created_event: + return created_event e = MISPEvent() e.load(created_event) return e @@ -105,6 +107,8 @@ class ExpandedPyMISP(PyMISP): updated_event = super().update_event(event.uuid, event) if isinstance(updated_event, str): raise UpdateEventError(f'Unexpected response from server: {updated_event}') + elif 'errors' in updated_event: + return updated_event e = MISPEvent() e.load(updated_event) return e @@ -113,6 +117,8 @@ class ExpandedPyMISP(PyMISP): updated_attribute = super().update_attribute(attribute.uuid, attribute) if isinstance(updated_attribute, str): raise UpdateAttributeError(f'Unexpected response from server: {updated_attribute}') + elif 'errors' in updated_attribute: + return updated_attribute a = MISPAttribute() a.from_dict(**updated_attribute) return a diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 9c532a3..e843d6f 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -469,8 +469,7 @@ class MISPEvent(AbstractMISP): 'attribute_count' in event.get('Event') and event.get('Event').get('attribute_count') is None): event['Event']['attribute_count'] = '0' - e = event.get('Event') - self.from_dict(**e) + self.from_dict(**event['Event']) if validate: jsonschema.validate(json.loads(self.to_json()), self.__json_schema) From 573e4a426cd3d6db61fbbc6f2ae2e84f187b80ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 29 Nov 2018 17:27:48 +0100 Subject: [PATCH 03/38] chg: Add test cases for default distribution levels --- pymisp/mispevent.py | 6 +++++ tests/testlive_comprehensive.py | 47 +++++++++++++++++++++++++++++++-- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index e843d6f..68263b7 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -1016,6 +1016,12 @@ class MISPObject(AbstractMISP): else: self._known_template = False + if 'distribution' in kwargs and kwargs['distribution'] is not None: + self.distribution = kwargs.pop('distribution') + self.distribution = int(self.distribution) + if self.distribution not in [0, 1, 2, 3, 4, 5]: + raise NewAttributeError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 4, 5'.format(self.distribution)) + if kwargs.get('timestamp'): if sys.version_info >= (3, 3): self.timestamp = datetime.datetime.fromtimestamp(int(kwargs.pop('timestamp')), datetime.timezone.utc) diff --git a/tests/testlive_comprehensive.py b/tests/testlive_comprehensive.py index b33976f..6b9d460 100644 --- a/tests/testlive_comprehensive.py +++ b/tests/testlive_comprehensive.py @@ -3,7 +3,7 @@ import unittest -from pymisp import ExpandedPyMISP, MISPEvent, MISPOrganisation, MISPUser, Distribution, ThreatLevel, Analysis +from pymisp import ExpandedPyMISP, MISPEvent, MISPOrganisation, MISPUser, Distribution, ThreatLevel, Analysis, MISPObject from datetime import datetime, timedelta, date from io import BytesIO @@ -14,7 +14,7 @@ try: except ImportError as e: print(e) url = 'http://localhost:8080' - key = 'BSip0zVadeFDeolkX2g7MHx8mrlr0uE04hh6CQj0' + key = 'LBelWqKY9SQyG0huZzAMqiEBl6FODxpgRRXMsZFu' from uuid import uuid4 @@ -439,6 +439,49 @@ class TestComprehensive(unittest.TestCase): self.admin_misp_connector.delete_event(first.id) self.admin_misp_connector.delete_event(second.id) + def test_default_distribution(self): + '''The default distributions on the VM are This community only for the events and Inherit from event for attr/obj)''' + first = self.create_simple_event() + del first.distribution + o = first.add_object(name='file') + o.add_attribute('filename', value='foo.exe') + try: + # Event create + first = self.user_misp_connector.add_event(first) + self.assertEqual(first.distribution, Distribution.this_community_only.value) + self.assertEqual(first.attributes[0].distribution, Distribution.inherit.value) + self.assertEqual(first.objects[0].distribution, Distribution.inherit.value) + self.assertEqual(first.objects[0].attributes[0].distribution, Distribution.inherit.value) + # Event edit + first.add_attribute('ip-dst', '12.54.76.43') + o = first.add_object(name='file') + o.add_attribute('filename', value='foo2.exe') + first = self.user_misp_connector.update_event(first) + self.assertEqual(first.attributes[1].distribution, Distribution.inherit.value) + self.assertEqual(first.objects[1].distribution, Distribution.inherit.value) + self.assertEqual(first.objects[1].attributes[0].distribution, Distribution.inherit.value) + # Attribute create + attribute = self.user_misp_connector.add_named_attribute(first, 'comment', 'bar') + # FIXME: Add helper that returns a list of MISPAttribute + self.assertEqual(attribute[0]['Attribute']['distribution'], str(Distribution.inherit.value)) + # Object - add + o = MISPObject('file') + o.add_attribute('filename', value='blah.exe') + new_obj = self.user_misp_connector.add_object(first.id, o.template_uuid, o) + # FIXME: Add helper that returns a MISPObject + self.assertEqual(new_obj['Object']['distribution'], str(Distribution.inherit.value)) + self.assertEqual(new_obj['Object']['Attribute'][0]['distribution'], str(Distribution.inherit.value)) + # Object - edit + clean_obj = MISPObject(**new_obj['Object']) + clean_obj.from_dict(**new_obj['Object']) + clean_obj.add_attribute('filename', value='blah.exe') + new_obj = self.user_misp_connector.edit_object(clean_obj) + for a in new_obj['Object']['Attribute']: + self.assertEqual(a['distribution'], str(Distribution.inherit.value)) + finally: + # Delete event + self.admin_misp_connector.delete_event(first.id) + def test_simple_event(self): '''Search a bunch of parameters: * Value not existing From b895c30b39cf79514f1f853e4aaa7c1e724dbd62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 29 Nov 2018 18:14:10 +0100 Subject: [PATCH 04/38] fix: Test failing on travis... --- tests/testlive_comprehensive.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/testlive_comprehensive.py b/tests/testlive_comprehensive.py index 6b9d460..f646e26 100644 --- a/tests/testlive_comprehensive.py +++ b/tests/testlive_comprehensive.py @@ -11,15 +11,15 @@ import time try: from keys import url, key + travis_run = True except ImportError as e: print(e) url = 'http://localhost:8080' key = 'LBelWqKY9SQyG0huZzAMqiEBl6FODxpgRRXMsZFu' + travis_run = False from uuid import uuid4 -travis_run = True - class TestComprehensive(unittest.TestCase): @@ -441,6 +441,8 @@ class TestComprehensive(unittest.TestCase): def test_default_distribution(self): '''The default distributions on the VM are This community only for the events and Inherit from event for attr/obj)''' + if travis_run: + return first = self.create_simple_event() del first.distribution o = first.add_object(name='file') @@ -560,7 +562,7 @@ class TestComprehensive(unittest.TestCase): # quickfilter events = self.user_misp_connector.search(timestamp=timeframe, - quickfilter='%bar%', pythonify=True) + quickfilter='%foo blah%', pythonify=True) # FIXME: should return one event # print(events) # self.assertEqual(len(events), 1) From cf622cd807ee7dc17b63870844747da9e6c6814e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 29 Nov 2018 18:14:55 +0100 Subject: [PATCH 05/38] chg: Bump misp-objects --- pymisp/data/misp-objects | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index 7fe77c0..6cc29aa 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit 7fe77c02affc0abe14cc67fe9f14400e8b72561c +Subproject commit 6cc29aad3dda895de95fe9f0d86bb9a7007af7c2 From 7e523fbff2626bda2afb4423510b487773740933 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 3 Dec 2018 17:08:49 +0100 Subject: [PATCH 06/38] chg: Version bump --- pymisp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index d613fa9..2ea8cd1 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.4.96' +__version__ = '2.4.98' import logging import functools import warnings From 49dcfb423dfa2e7b75d1d5b9c6d0d4b56d6d0cfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 3 Dec 2018 17:10:21 +0100 Subject: [PATCH 07/38] chg: Bump Changelog --- CHANGELOG.txt | 83 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 2e7a0c8..35ac554 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -2,6 +2,88 @@ Changelog ========= +v2.4.98 (2018-12-03) +-------------------- + +New +~~~ +- Search_index in ExpandedPyMISP, cleanup, update jupyter. [Raphaël + Vinot] +- Add log search. [Raphaël Vinot] +- Add test for pushing an event to ZMQ. [Raphaël Vinot] +- Change_distribution method. [Raphaël Vinot] +- Add test cases for sightings, cleanup. [Raphaël Vinot] +- [example] Added sighting rest search example. [Sami Mokaddem] +- [sighting] Added support of sighting REST API. [Sami Mokaddem] +- Allow to pass csv to return_format in search. [Raphaël Vinot] +- Page/limit in search. [Raphaël Vinot] + +Changes +~~~~~~~ +- Version bump. [Raphaël Vinot] +- Bump misp-objects. [Raphaël Vinot] +- Add test cases for default distribution levels. [Raphaël Vinot] +- Include proposals in attributes search. [Dawid Czarnecki] + + Add includeProposals param to the search method +- Bump misp-objects. [Raphaël Vinot] +- Update readme to document testing. [Raphaël Vinot] +- Fixes & update Jupyter. [Raphaël Vinot] +- [tuto] Update search. [Raphaël Vinot] +- Add a script to load the API key from the file system (training VM) + [Raphaël Vinot] +- Bump misp-objects. [Raphaël Vinot] +- Add print in testlive to debug travis. [Raphaël Vinot] +- Bump objects. [Raphaël Vinot] + +Fix +~~~ +- Test failing on travis... [Raphaël Vinot] +- Properly handle errors on event creation/update. [Raphaël Vinot] +- Test case. [Raphaël Vinot] +- Do not run the zmq test on travis. [Raphaël Vinot] +- Type of quick_filter. [Raphaël Vinot] +- Quick_filter was broken. [Raphaël Vinot] +- Properly initialize the config when jupyter runs on the VM. [Raphaël + Vinot] +- Travis run. [Raphaël Vinot] +- Readme update + python3 + pep8. [Christophe Vandeplas] + + align python path to readme specifying python3 +- Feed-generator gitignore. [Christophe Vandeplas] +- Test cases. [Raphaël Vinot] +- Test cases sample files. [Raphaël Vinot] + +Other +~~~~~ +- Merge pull request #305 from dawid- + czarnecki/feature/include_proposals. [Raphaël Vinot] + + chg: Include proposals in attributes search +- Merge pull request #301 from deralexxx/patch-7. [Raphaël Vinot] + + mention virtualenv +- Mention virtualenv. [Alexander J] + + mide make sense for people who want to use it with virtualenv +- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot] +- Be more precise with the supported time indicators. [Sascha + Rommelfangen] +- Fixed documentation bug. [Sascha Rommelfangen] +- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot] +- Merge pull request #295 from 3c7/fix/search_index_date. [Raphaël + Vinot] + + Fixes date parameters for search_index() function +- Fixes date parameters for search_index() function. [Nils Kuhnert] +- Merge branch 'sightingAPI' [Raphaël Vinot] +- Merge branch 'master' into sightingAPI. [Raphaël Vinot] +- Merge pull request #285 from juju4/devel. [Raphaël Vinot] + + align examples on custom usage of misp_verifycert +- Align examples on custom usage of misp_verifycert. [juju4] + + v2.4.96 (2018-10-12) -------------------- @@ -22,6 +104,7 @@ New Changes ~~~~~~~ +- Bump changelog. [Raphaël Vinot] - Bump version. [Raphaël Vinot] - Bump misp-objects. [Raphaël Vinot] - Allow to pass a json string to direct_call. [Raphaël Vinot] From af4ce20b5be0680b80bdbd2913169e4007421cbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 3 Dec 2018 17:29:24 +0100 Subject: [PATCH 08/38] new: Auto generate doc for PyMISPExpanded --- docs/source/modules.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/source/modules.rst b/docs/source/modules.rst index 7db7414..9472f8c 100644 --- a/docs/source/modules.rst +++ b/docs/source/modules.rst @@ -14,6 +14,13 @@ PyMISP .. autoclass:: PyMISP :members: +PyMISPExpanded (Python 3.6+ only) +------ + +.. autoclass:: PyMISPExpanded + :members: + +MISPAbstract MISPAbstract ------------ From 98feed73734f4577449ee584232015137feb0031 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 3 Dec 2018 17:39:26 +0100 Subject: [PATCH 09/38] fix: Auto generate doc for PyMISPExpanded --- docs/source/modules.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/source/modules.rst b/docs/source/modules.rst index 9472f8c..5fc2210 100644 --- a/docs/source/modules.rst +++ b/docs/source/modules.rst @@ -15,12 +15,11 @@ PyMISP :members: PyMISPExpanded (Python 3.6+ only) ------- +--------------------------------- .. autoclass:: PyMISPExpanded :members: -MISPAbstract MISPAbstract ------------ From 35b6fc3cb52ca5963f6ae9f29b10fc04b2feb07d Mon Sep 17 00:00:00 2001 From: garanews Date: Tue, 4 Dec 2018 16:08:00 +0000 Subject: [PATCH 10/38] fix for last pymisp version --- examples/add_generic_object.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/examples/add_generic_object.py b/examples/add_generic_object.py index 36e04cd..86a7675 100755 --- a/examples/add_generic_object.py +++ b/examples/add_generic_object.py @@ -20,10 +20,17 @@ if __name__ == '__main__': args = parser.parse_args() pymisp = PyMISP(misp_url, misp_key, misp_verifycert) + template = pymisp.get_object_templates_list() + if 'response' in template.keys(): + template = template['response'] try: - template_id = [x['ObjectTemplate']['id'] for x in pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == args.type][0] + template_ids = [x['ObjectTemplate']['id'] for x in template if x['ObjectTemplate']['name'] == args.type] + if len(template_ids) > 0: + template_id = template_ids[0] + else: + raise IndexError except IndexError: - valid_types = ", ".join([x['ObjectTemplate']['name'] for x in pymisp.get_object_templates_list()]) + valid_types = ", ".join([x['ObjectTemplate']['name'] for x in template]) print ("Template for type %s not found! Valid types are: %s" % (args.type, valid_types)) exit() From 5c72dc9c33b4ae850d40ff06dfb05c27f3e80e5d Mon Sep 17 00:00:00 2001 From: DragonDev1906 Date: Thu, 6 Dec 2018 14:26:23 +0100 Subject: [PATCH 11/38] dded get_object & get_attribute --- pymisp/api.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pymisp/api.py b/pymisp/api.py index 4e0f8a5..a59c99a 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -336,6 +336,24 @@ class PyMISP(object): response = self._prepare_request('GET', url) return self._check_response(response) + def get_object(self, obj_id): + """Get an object + + :param obj_id: Object id to get + """ + url = urljoin(self.root_url, 'objects/view/{}'.format(obj_id)) + response = self._prepare_request('GET', url) + return self._check_response(response) + + def get_attribute(self, att_id): + """Get an attribute + + :param att_id: Attribute id to get + """ + url = urljoin(self.root_url, 'attributes/view/{}'.format(att_id)) + response = self._prepare_request('GET', url) + return self._check_response(response) + def add_event(self, event): """Add a new event From 8fd4da1b803d95899014f8451c28943ca4006eba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 6 Dec 2018 15:16:22 +0100 Subject: [PATCH 12/38] chg: Bump misp-objects & describeTypes --- pymisp/data/describeTypes.json | 288 +++++++++++++++++---------------- pymisp/data/misp-objects | 2 +- 2 files changed, 146 insertions(+), 144 deletions(-) diff --git a/pymisp/data/describeTypes.json b/pymisp/data/describeTypes.json index 497c44b..cb90814 100644 --- a/pymisp/data/describeTypes.json +++ b/pymisp/data/describeTypes.json @@ -1,22 +1,22 @@ { "result": { "categories": [ + "Internal reference", + "Targeting data", "Antivirus detection", + "Payload delivery", "Artifacts dropped", + "Payload installation", + "Persistence mechanism", + "Network activity", + "Payload type", "Attribution", "External analysis", "Financial fraud", - "Internal reference", - "Network activity", - "Other", - "Payload delivery", - "Payload installation", - "Payload type", - "Persistence mechanism", - "Person", - "Social network", "Support Tool", - "Targeting data" + "Social network", + "Person", + "Other" ], "category_type_mappings": { "Antivirus detection": [ @@ -186,7 +186,9 @@ "attachment", "comment", "text", + "x509-fingerprint-md5", "x509-fingerprint-sha1", + "x509-fingerprint-sha256", "other", "hex", "cookie", @@ -1019,158 +1021,158 @@ } }, "types": [ - "AS", - "aba-rtn", - "attachment", - "authentihash", - "bank-account-nr", - "bic", - "bin", - "boolean", - "bro", - "btc", - "campaign-id", - "campaign-name", - "cc-number", - "comment", - "cookie", - "cortex", - "counter", - "country-of-residence", - "cpe", - "date-of-birth", - "datetime", - "dns-soa-email", + "md5", + "sha1", + "sha256", + "filename", + "pdb", + "filename|md5", + "filename|sha1", + "filename|sha256", + "ip-src", + "ip-dst", + "hostname", "domain", "domain|ip", + "email-src", + "email-dst", + "email-subject", "email-attachment", "email-body", - "email-dst", - "email-dst-display-name", - "email-header", - "email-message-id", - "email-mime-boundary", - "email-reply-to", - "email-src", - "email-src-display-name", - "email-subject", - "email-thread-index", - "email-x-mailer", - "filename", - "filename|authentihash", - "filename|impfuzzy", - "filename|imphash", - "filename|md5", - "filename|pehash", - "filename|sha1", - "filename|sha224", - "filename|sha256", - "filename|sha384", - "filename|sha512", - "filename|sha512/224", - "filename|sha512/256", - "filename|ssdeep", - "filename|tlsh", - "first-name", "float", - "frequent-flyer-number", - "gender", - "gene", - "github-organisation", - "github-repository", - "github-username", - "hex", - "hostname", - "hostname|port", + "url", "http-method", - "iban", - "identity-card-number", - "impfuzzy", - "imphash", - "ip-dst", - "ip-dst|port", - "ip-src", - "ip-src|port", - "issue-date-of-the-visa", - "jabber-id", - "last-name", - "link", - "mac-address", - "mac-eui-64", - "malware-sample", - "malware-type", - "md5", - "middle-name", - "mime-type", - "mobile-application-id", - "mutex", - "named pipe", - "nationality", - "other", - "passenger-name-record-locator-number", - "passport-country", - "passport-expiration", - "passport-number", - "pattern-in-file", - "pattern-in-memory", - "pattern-in-traffic", - "payment-details", - "pdb", - "pehash", - "phone-number", - "place-of-birth", - "place-port-of-clearance", - "place-port-of-onward-foreign-destination", - "place-port-of-original-embarkation", - "port", - "primary-residence", - "prtn", - "redress-number", + "user-agent", "regkey", "regkey|value", - "sha1", + "AS", + "snort", + "bro", + "pattern-in-file", + "pattern-in-traffic", + "pattern-in-memory", + "yara", + "stix2-pattern", + "sigma", + "gene", + "mime-type", + "identity-card-number", + "cookie", + "vulnerability", + "attachment", + "malware-sample", + "link", + "comment", + "text", + "hex", + "other", + "named pipe", + "mutex", + "target-user", + "target-email", + "target-machine", + "target-org", + "target-location", + "target-external", + "btc", + "xmr", + "iban", + "bic", + "bank-account-nr", + "aba-rtn", + "bin", + "cc-number", + "prtn", + "phone-number", + "threat-actor", + "campaign-name", + "campaign-id", + "malware-type", + "uri", + "authentihash", + "ssdeep", + "imphash", + "pehash", + "impfuzzy", "sha224", - "sha256", "sha384", "sha512", "sha512/224", "sha512/256", - "sigma", - "size-in-bytes", - "snort", - "special-service-request", - "ssdeep", - "stix2-pattern", - "target-email", - "target-external", - "target-location", - "target-machine", - "target-org", - "target-user", - "text", - "threat-actor", "tlsh", - "travel-details", - "twitter-id", - "uri", - "url", - "user-agent", - "visa-number", - "vulnerability", - "whois-creation-date", + "filename|authentihash", + "filename|ssdeep", + "filename|imphash", + "filename|impfuzzy", + "filename|pehash", + "filename|sha224", + "filename|sha384", + "filename|sha512", + "filename|sha512/224", + "filename|sha512/256", + "filename|tlsh", + "windows-scheduled-task", + "windows-service-name", + "windows-service-displayname", "whois-registrant-email", + "whois-registrant-phone", "whois-registrant-name", "whois-registrant-org", - "whois-registrant-phone", "whois-registrar", - "windows-scheduled-task", - "windows-service-displayname", - "windows-service-name", - "x509-fingerprint-md5", + "whois-creation-date", "x509-fingerprint-sha1", + "x509-fingerprint-md5", "x509-fingerprint-sha256", - "xmr", - "yara" + "dns-soa-email", + "size-in-bytes", + "counter", + "datetime", + "cpe", + "port", + "ip-dst|port", + "ip-src|port", + "hostname|port", + "mac-address", + "mac-eui-64", + "email-dst-display-name", + "email-src-display-name", + "email-header", + "email-reply-to", + "email-x-mailer", + "email-mime-boundary", + "email-thread-index", + "email-message-id", + "github-username", + "github-repository", + "github-organisation", + "jabber-id", + "twitter-id", + "first-name", + "middle-name", + "last-name", + "date-of-birth", + "place-of-birth", + "gender", + "passport-number", + "passport-country", + "passport-expiration", + "redress-number", + "nationality", + "visa-number", + "issue-date-of-the-visa", + "primary-residence", + "country-of-residence", + "special-service-request", + "frequent-flyer-number", + "travel-details", + "payment-details", + "place-port-of-original-embarkation", + "place-port-of-clearance", + "place-port-of-onward-foreign-destination", + "passenger-name-record-locator-number", + "mobile-application-id", + "cortex", + "boolean" ] } -} \ No newline at end of file +} diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index 6cc29aa..11a462e 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit 6cc29aad3dda895de95fe9f0d86bb9a7007af7c2 +Subproject commit 11a462e79b02428a08b11698d45aa8aa5ab6887d From 93e985d2b66032210d9bd4a7202842ac1064704b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 6 Dec 2018 15:17:03 +0100 Subject: [PATCH 13/38] chg: Bump version --- pymisp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 2ea8cd1..09a3d52 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.4.98' +__version__ = '2.4.99' import logging import functools import warnings From ab908af5a7106f7c8791149b7918a7b952802e7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 6 Dec 2018 15:18:39 +0100 Subject: [PATCH 14/38] chg: Bump Changelog --- CHANGELOG.txt | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 35ac554..12671d7 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -2,11 +2,12 @@ Changelog ========= -v2.4.98 (2018-12-03) +v2.4.99 (2018-12-06) -------------------- New ~~~ +- Auto generate doc for PyMISPExpanded. [Raphaël Vinot] - Search_index in ExpandedPyMISP, cleanup, update jupyter. [Raphaël Vinot] - Add log search. [Raphaël Vinot] @@ -20,6 +21,9 @@ New Changes ~~~~~~~ +- Bump version. [Raphaël Vinot] +- Bump misp-objects & describeTypes. [Raphaël Vinot] +- Bump Changelog. [Raphaël Vinot] - Version bump. [Raphaël Vinot] - Bump misp-objects. [Raphaël Vinot] - Add test cases for default distribution levels. [Raphaël Vinot] @@ -38,6 +42,7 @@ Changes Fix ~~~ +- Auto generate doc for PyMISPExpanded. [Raphaël Vinot] - Test failing on travis... [Raphaël Vinot] - Properly handle errors on event creation/update. [Raphaël Vinot] - Test case. [Raphaël Vinot] @@ -52,10 +57,10 @@ Fix align python path to readme specifying python3 - Feed-generator gitignore. [Christophe Vandeplas] - Test cases. [Raphaël Vinot] -- Test cases sample files. [Raphaël Vinot] Other ~~~~~ +- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot] - Merge pull request #305 from dawid- czarnecki/feature/include_proposals. [Raphaël Vinot] @@ -115,6 +120,7 @@ Changes Fix ~~~ +- Test cases sample files. [Raphaël Vinot] - Prevent checking length on a integer. [Sami Mokaddem] - Direct call & add example. [Raphaël Vinot] - Disable test for travis, take 2. [Raphaël Vinot] From a2388bb242f0774310ec4f8d3350e86efe51770a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 6 Dec 2018 15:20:25 +0100 Subject: [PATCH 15/38] chg: Bump Changelog, again --- CHANGELOG.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 12671d7..d8c1dc2 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -21,6 +21,7 @@ New Changes ~~~~~~~ +- Bump Changelog. [Raphaël Vinot] - Bump version. [Raphaël Vinot] - Bump misp-objects & describeTypes. [Raphaël Vinot] - Bump Changelog. [Raphaël Vinot] @@ -61,6 +62,15 @@ Fix Other ~~~~~ - Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot] +- Merge pull request #310 from DragonDev1906/master. [Raphaël Vinot] + + Added get_object & get_attribute (by ID) +- Dded get_object & get_attribute. [DragonDev1906] +- Merge pull request #307 from garanews/patch-1. [Raphaël Vinot] + + fix for last pymisp version +- Fix for last pymisp version. [garanews] +- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot] - Merge pull request #305 from dawid- czarnecki/feature/include_proposals. [Raphaël Vinot] From 486017d345dbb5dd127d06a2e9ae28336604c928 Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Sun, 9 Dec 2018 13:26:43 +0100 Subject: [PATCH 16/38] fix: get_object_template_id --- pymisp/api.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index a59c99a..8e81ca8 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -2282,11 +2282,9 @@ class PyMISP(object): def get_object_template_id(self, object_uuid): """Gets the template ID corresponting the UUID passed as parameter""" - templates = self.get_object_templates_list() - for t in templates: - if t['ObjectTemplate']['uuid'] == object_uuid: - return t['ObjectTemplate']['id'] - raise Exception('Unable to find template uuid {} on the MISP instance'.format(object_uuid)) + url = urljoin(self.root_url, 'objectTemplates/view/{}'.format(object_uuid)) + response = self._prepare_request('GET', url) + return self._check_response(response) def update_object_templates(self): url = urljoin(self.root_url, '/objectTemplates/update') From 58d08889b3ee25bdfd82e6ced3cf765ca7d92b54 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Mon, 10 Dec 2018 12:58:59 +0100 Subject: [PATCH 17/38] chg: [tests] Added verifycert option in case of using self-signed cert. --- tests/testlive_comprehensive.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/testlive_comprehensive.py b/tests/testlive_comprehensive.py index f646e26..3904e4b 100644 --- a/tests/testlive_comprehensive.py +++ b/tests/testlive_comprehensive.py @@ -16,6 +16,7 @@ except ImportError as e: print(e) url = 'http://localhost:8080' key = 'LBelWqKY9SQyG0huZzAMqiEBl6FODxpgRRXMsZFu' + verifycert = False travis_run = False from uuid import uuid4 @@ -27,7 +28,7 @@ class TestComprehensive(unittest.TestCase): def setUpClass(cls): cls.maxDiff = None # Connect as admin - cls.admin_misp_connector = ExpandedPyMISP(url, key, debug=False) + cls.admin_misp_connector = ExpandedPyMISP(url, key, verifycert, debug=False) # Creates an org org = cls.admin_misp_connector.add_organisation(name='Test Org') cls.test_org = MISPOrganisation() @@ -36,12 +37,12 @@ class TestComprehensive(unittest.TestCase): usr = cls.admin_misp_connector.add_user(email='testusr@user.local', org_id=cls.test_org.id, role_id=3) cls.test_usr = MISPUser() cls.test_usr.from_dict(**usr) - cls.user_misp_connector = ExpandedPyMISP(url, cls.test_usr.authkey) + cls.user_misp_connector = ExpandedPyMISP(url, cls.test_usr.authkey, verifycert) # Creates a publisher pub = cls.admin_misp_connector.add_user(email='testpub@user.local', org_id=cls.test_org.id, role_id=4) cls.test_pub = MISPUser() cls.test_pub.from_dict(**pub) - cls.pub_misp_connector = ExpandedPyMISP(url, cls.test_pub.authkey) + cls.pub_misp_connector = ExpandedPyMISP(url, cls.test_pub.authkey, verifycert) @classmethod def tearDownClass(cls): From 864aa5fe70b43dbc8bfd0731ce30051c7da764ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 10 Dec 2018 14:08:23 +0100 Subject: [PATCH 18/38] chg: set verifycert to false in tests --- tests/testlive_comprehensive.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/testlive_comprehensive.py b/tests/testlive_comprehensive.py index 3904e4b..d7837ea 100644 --- a/tests/testlive_comprehensive.py +++ b/tests/testlive_comprehensive.py @@ -8,9 +8,11 @@ from datetime import datetime, timedelta, date from io import BytesIO import time +from uuid import uuid4 try: from keys import url, key + verifycert = False travis_run = True except ImportError as e: print(e) @@ -19,8 +21,6 @@ except ImportError as e: verifycert = False travis_run = False -from uuid import uuid4 - class TestComprehensive(unittest.TestCase): From a09915d8507f41f8ebd12caf6f7cab4b9471f428 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 18 Dec 2018 11:04:36 +0100 Subject: [PATCH 19/38] chg: Remove compat for MISP 2.4.52, cleanup. --- pymisp/__init__.py | 2 +- pymisp/api.py | 9 ++++++++- pymisp/aping.py | 29 ++++++++++------------------- pymisp/exceptions.py | 4 ++++ pymisp/mispevent.py | 9 +++------ 5 files changed, 26 insertions(+), 27 deletions(-) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 09a3d52..54800cf 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -32,7 +32,7 @@ def deprecated(func): try: - from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate, PyMISPInvalidFormat, MISPServerError, PyMISPNotImplementedYet, PyMISPUnexpectedResponse # noqa + from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate, PyMISPInvalidFormat, MISPServerError, PyMISPNotImplementedYet, PyMISPUnexpectedResponse, PyMISPEmptyResponse # noqa from .api import PyMISP # noqa from .abstract import AbstractMISP, MISPEncode, MISPTag, Distribution, ThreatLevel, Analysis # noqa from .mispevent import MISPEvent, MISPAttribute, MISPObjectReference, MISPObjectAttribute, MISPObject, MISPUser, MISPOrganisation, MISPSighting, MISPLog # noqa diff --git a/pymisp/api.py b/pymisp/api.py index 8e81ca8..d7bfd97 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -15,7 +15,7 @@ from io import BytesIO, open import zipfile from . import __version__, deprecated -from .exceptions import PyMISPError, SearchError, NoURL, NoKey +from .exceptions import PyMISPError, SearchError, NoURL, NoKey, PyMISPEmptyResponse from .mispevent import MISPEvent, MISPAttribute, MISPUser, MISPOrganisation, MISPSighting, MISPFeed, MISPObject from .abstract import AbstractMISP, MISPEncode @@ -157,6 +157,11 @@ class PyMISP(object): if data is None: req = requests.Request(request_type, url) else: + if not isinstance(data, str): + if isinstance(data, dict): + # Remove None values. + data = {k: v for k, v in data.items() if v is not None} + data = json.dumps(data) req = requests.Request(request_type, url, data=data) if self.asynch and background_callback is not None: local_session = FuturesSession @@ -227,6 +232,8 @@ class PyMISP(object): json_response = response.json() except ValueError: # If the server didn't return a JSON blob, we've a problem. + if not len(response.text): + raise PyMISPEmptyResponse('The server returned an empty response. \n{}\n{}\n'.format(response.request.headers, response.request.body)) raise PyMISPError(everything_broken.format(response.request.headers, response.request.body, response.text)) errors = [] diff --git a/pymisp/aping.py b/pymisp/aping.py index 9f59273..88d8a6b 100644 --- a/pymisp/aping.py +++ b/pymisp/aping.py @@ -6,7 +6,6 @@ from .api import PyMISP, everything_broken from .mispevent import MISPEvent, MISPAttribute, MISPSighting, MISPLog from typing import TypeVar, Optional, Tuple, List, Dict from datetime import date, datetime -import json import csv import logging @@ -70,7 +69,7 @@ class ExpandedPyMISP(PyMISP): # The server returns a json message with the error details error_message = response.json() logger.error(f'Something went wrong ({response.status_code}): {error_message}') - return {'errors': [(response.status_code, error_message)]} + return {'error': (response.status_code, error_message)} # At this point, we had no error. @@ -80,11 +79,15 @@ class ExpandedPyMISP(PyMISP): logger.debug(response) if isinstance(response, dict) and response.get('response') is not None: # Cleanup. - return response.get('response') + response = response['response'] return response except Exception: if logger.isEnabledFor(logging.DEBUG): logger.debug(response.text) + if not len(response.content): + # Empty response + logger.error('Got an empty response.') + return {'error': 'The response is empty.'} return response.text def get_event(self, event_id: int): @@ -178,10 +181,7 @@ class ExpandedPyMISP(PyMISP): query['includeEvent'] = include_event_meta url = urljoin(self.root_url, url_path) - # Remove None values. - # TODO: put that in self._prepare_request - query = {k: v for k, v in query.items() if v is not None} - response = self._prepare_request('POST', url, data=json.dumps(query)) + response = self._prepare_request('POST', url, data=query) normalized_response = self._check_response(response) if isinstance(normalized_response, str) or (isinstance(normalized_response, dict) and normalized_response.get('errors')): @@ -354,10 +354,7 @@ class ExpandedPyMISP(PyMISP): query['includeContext'] = include_context query['headerless'] = headerless url = urljoin(self.root_url, f'{controller}/restSearch') - # Remove None values. - # TODO: put that in self._prepare_request - query = {k: v for k, v in query.items() if v is not None} - response = self._prepare_request('POST', url, data=json.dumps(query)) + response = self._prepare_request('POST', url, data=query) normalized_response = self._check_response(response) if return_format == 'csv' and pythonify and not headerless: return self._csv_to_dict(normalized_response) @@ -426,10 +423,7 @@ class ExpandedPyMISP(PyMISP): query['id'] = query.pop('log_id') url = urljoin(self.root_url, 'admin/logs/index') - # Remove None values. - # TODO: put that in self._prepare_request - query = {k: v for k, v in query.items() if v is not None} - response = self._prepare_request('POST', url, data=json.dumps(query)) + response = self._prepare_request('POST', url, data=query) normalized_response = self._check_response(response) if not pythonify: return normalized_response @@ -483,10 +477,7 @@ class ExpandedPyMISP(PyMISP): query['timestamp'] = self.make_timestamp(timestamp) url = urljoin(self.root_url, 'events/index') - # Remove None values. - # TODO: put that in self._prepare_request - query = {k: v for k, v in query.items() if v is not None} - response = self._prepare_request('POST', url, data=json.dumps(query)) + response = self._prepare_request('POST', url, data=query) normalized_response = self._check_response(response) if not pythonify: diff --git a/pymisp/exceptions.py b/pymisp/exceptions.py index 481720b..1d7663f 100644 --- a/pymisp/exceptions.py +++ b/pymisp/exceptions.py @@ -67,3 +67,7 @@ class PyMISPNotImplementedYet(PyMISPError): class PyMISPUnexpectedResponse(PyMISPError): pass + + +class PyMISPEmptyResponse(PyMISPError): + pass diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 68263b7..a74794c 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -464,12 +464,7 @@ class MISPEvent(AbstractMISP): event = json_event if not event: raise PyMISPError('Invalid event') - # Invalid event created by MISP up to 2.4.52 (attribute_count is none instead of '0') - if (event.get('Event') and - 'attribute_count' in event.get('Event') and - event.get('Event').get('attribute_count') is None): - event['Event']['attribute_count'] = '0' - self.from_dict(**event['Event']) + self.from_dict(**event) if validate: jsonschema.validate(json.loads(self.to_json()), self.__json_schema) @@ -488,6 +483,8 @@ class MISPEvent(AbstractMISP): raise NewEventError('Invalid format for the date: {} - {}'.format(date, type(date))) def from_dict(self, **kwargs): + if kwargs.get('Event'): + kwargs = kwargs.get('Event') # Required value self.info = kwargs.pop('info', None) if self.info is None: From 5de57816ddeac42cfee3780d3e0d68f023ede272 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 18 Dec 2018 15:29:40 +0100 Subject: [PATCH 20/38] chg: Pass all parameters to the search API. --- pymisp/api.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 65174e7..b66eae0 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1231,12 +1231,14 @@ class PyMISP(object): query['event_timestamp'] = kwargs.pop('event_timestamp', None) query['includeProposals'] = kwargs.pop('includeProposals', None) + if kwargs: + self.logger.info('Some unknown parameters are in kwargs. appending as-is: {}'.format(', '.join(kwargs.keys()))) + # Add all other keys as-is. + query.update({k: v for k, v in kwargs.items()}) + # Cleanup query = {k: v for k, v in query.items() if v is not None} - if kwargs: - raise SearchError('Unused parameter: {}'.format(', '.join(kwargs.keys()))) - # Create a session, make it async if and only if we have a callback return self.__query('restSearch/download', query, controller, async_callback) From 9365f801d473e662c651cc1d0ac161cd3bbc63a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 19 Dec 2018 10:50:50 +0100 Subject: [PATCH 21/38] fix: Typo --- pymisp/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/api.py b/pymisp/api.py index b66eae0..6e2b7ca 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1232,7 +1232,7 @@ class PyMISP(object): query['includeProposals'] = kwargs.pop('includeProposals', None) if kwargs: - self.logger.info('Some unknown parameters are in kwargs. appending as-is: {}'.format(', '.join(kwargs.keys()))) + logger.info('Some unknown parameters are in kwargs. appending as-is: {}'.format(', '.join(kwargs.keys()))) # Add all other keys as-is. query.update({k: v for k, v in kwargs.items()}) From b72aa536aa39df8e422b050c4cbf0cf17f40c29c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 19 Dec 2018 11:44:42 +0100 Subject: [PATCH 22/38] chg: More flexibility when loading an object from python dict --- pymisp/mispevent.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index a74794c..18cd938 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -1001,6 +1001,8 @@ class MISPObject(AbstractMISP): raise PyMISPError('All the attributes have to be of type MISPObjectReference.') def from_dict(self, **kwargs): + if kwargs.get('Object'): + kwargs = kwargs.get('Object') if self._known_template: if kwargs.get('template_uuid') and kwargs['template_uuid'] != self.template_uuid: if self._strict: From 268241cb4fb0134a597052d92e16a9e0142cf722 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 20 Dec 2018 12:05:46 +0100 Subject: [PATCH 23/38] new: Bump describe types fix #317 --- pymisp/data/describeTypes.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pymisp/data/describeTypes.json b/pymisp/data/describeTypes.json index cb90814..1d4de59 100644 --- a/pymisp/data/describeTypes.json +++ b/pymisp/data/describeTypes.json @@ -40,6 +40,7 @@ "imphash", "impfuzzy", "authentihash", + "cdhash", "filename", "filename|md5", "filename|sha1", @@ -224,6 +225,7 @@ "authentihash", "pehash", "tlsh", + "cdhash", "filename", "filename|md5", "filename|sha1", @@ -300,6 +302,7 @@ "authentihash", "pehash", "tlsh", + "cdhash", "filename", "filename|md5", "filename|sha1", @@ -463,6 +466,10 @@ "default_category": "Financial fraud", "to_ids": 1 }, + "cdhash": { + "default_category": "Payload delivery", + "to_ids": 1 + }, "comment": { "default_category": "Other", "to_ids": 0 @@ -1100,6 +1107,7 @@ "sha512/224", "sha512/256", "tlsh", + "cdhash", "filename|authentihash", "filename|ssdeep", "filename|imphash", From bfbaa70ee3a370fed076b650e110db7083f83f9c Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Thu, 20 Dec 2018 13:24:08 +0100 Subject: [PATCH 24/38] sort describeTypes.json output This is needed for the compatibility with the gen_misp_types_categories.py script. Data was sorted using the order_dict function of the gen_misp_types_categories script. --- pymisp/data/describeTypes.json | 816 ++++++++++++++++----------------- 1 file changed, 408 insertions(+), 408 deletions(-) diff --git a/pymisp/data/describeTypes.json b/pymisp/data/describeTypes.json index 1d4de59..6db79e1 100644 --- a/pymisp/data/describeTypes.json +++ b/pymisp/data/describeTypes.json @@ -1,48 +1,44 @@ { "result": { "categories": [ - "Internal reference", - "Targeting data", "Antivirus detection", - "Payload delivery", "Artifacts dropped", - "Payload installation", - "Persistence mechanism", - "Network activity", - "Payload type", "Attribution", "External analysis", "Financial fraud", - "Support Tool", - "Social network", + "Internal reference", + "Network activity", + "Other", + "Payload delivery", + "Payload installation", + "Payload type", + "Persistence mechanism", "Person", - "Other" + "Social network", + "Support Tool", + "Targeting data" ], "category_type_mappings": { "Antivirus detection": [ - "link", - "comment", - "text", - "hex", "attachment", - "other" + "comment", + "hex", + "link", + "other", + "text" ], "Artifacts dropped": [ - "md5", - "sha1", - "sha224", - "sha256", - "sha384", - "sha512", - "sha512/224", - "sha512/256", - "ssdeep", - "imphash", - "impfuzzy", + "attachment", "authentihash", "cdhash", + "comment", + "cookie", "filename", + "filename|authentihash", + "filename|impfuzzy", + "filename|imphash", "filename|md5", + "filename|pehash", "filename|sha1", "filename|sha224", "filename|sha256", @@ -50,261 +46,196 @@ "filename|sha512", "filename|sha512/224", "filename|sha512/256", - "filename|authentihash", "filename|ssdeep", "filename|tlsh", - "filename|imphash", - "filename|impfuzzy", - "filename|pehash", - "regkey", - "regkey|value", + "gene", + "hex", + "impfuzzy", + "imphash", + "malware-sample", + "md5", + "mime-type", + "mutex", + "named pipe", + "other", "pattern-in-file", "pattern-in-memory", "pdb", - "stix2-pattern", - "yara", + "regkey", + "regkey|value", + "sha1", + "sha224", + "sha256", + "sha384", + "sha512", + "sha512/224", + "sha512/256", "sigma", - "attachment", - "malware-sample", - "named pipe", - "mutex", - "windows-scheduled-task", - "windows-service-name", - "windows-service-displayname", - "comment", + "ssdeep", + "stix2-pattern", "text", - "hex", - "x509-fingerprint-sha1", + "windows-scheduled-task", + "windows-service-displayname", + "windows-service-name", "x509-fingerprint-md5", + "x509-fingerprint-sha1", "x509-fingerprint-sha256", - "other", - "cookie", - "gene", - "mime-type" + "yara" ], "Attribution": [ - "threat-actor", - "campaign-name", "campaign-id", - "whois-registrant-phone", + "campaign-name", + "comment", + "dns-soa-email", + "other", + "text", + "threat-actor", + "whois-creation-date", "whois-registrant-email", "whois-registrant-name", "whois-registrant-org", + "whois-registrant-phone", "whois-registrar", - "whois-creation-date", - "comment", - "text", - "x509-fingerprint-sha1", "x509-fingerprint-md5", - "x509-fingerprint-sha256", - "other", - "dns-soa-email" + "x509-fingerprint-sha1", + "x509-fingerprint-sha256" ], "External analysis": [ - "md5", - "sha1", - "sha256", + "AS", + "attachment", + "bro", + "comment", + "cortex", + "domain", + "domain|ip", "filename", "filename|md5", "filename|sha1", "filename|sha256", - "ip-src", + "github-repository", + "hostname", "ip-dst", "ip-dst|port", + "ip-src", "ip-src|port", + "link", "mac-address", "mac-eui-64", - "hostname", - "domain", - "domain|ip", - "url", - "user-agent", + "malware-sample", + "md5", + "other", + "pattern-in-file", + "pattern-in-memory", + "pattern-in-traffic", "regkey", "regkey|value", - "AS", + "sha1", + "sha256", "snort", - "bro", - "pattern-in-file", - "pattern-in-traffic", - "pattern-in-memory", - "vulnerability", - "attachment", - "malware-sample", - "link", - "comment", "text", - "x509-fingerprint-sha1", + "url", + "user-agent", + "vulnerability", "x509-fingerprint-md5", - "x509-fingerprint-sha256", - "github-repository", - "other", - "cortex" + "x509-fingerprint-sha1", + "x509-fingerprint-sha256" ], "Financial fraud": [ - "btc", - "xmr", - "iban", - "bic", - "bank-account-nr", "aba-rtn", + "bank-account-nr", + "bic", "bin", + "btc", "cc-number", - "prtn", - "phone-number", "comment", - "text", + "hex", + "iban", "other", - "hex" + "phone-number", + "prtn", + "text", + "xmr" ], "Internal reference": [ - "text", - "link", "comment", + "hex", + "link", "other", - "hex" + "text" ], "Network activity": [ - "ip-src", - "ip-dst", - "ip-dst|port", - "ip-src|port", - "port", - "hostname", + "AS", + "attachment", + "bro", + "comment", + "cookie", "domain", "domain|ip", - "mac-address", - "mac-eui-64", "email-dst", - "url", - "uri", - "user-agent", - "http-method", - "AS", - "snort", - "pattern-in-file", - "stix2-pattern", - "pattern-in-traffic", - "attachment", - "comment", - "text", - "x509-fingerprint-md5", - "x509-fingerprint-sha1", - "x509-fingerprint-sha256", - "other", "hex", - "cookie", + "hostname", "hostname|port", - "bro" - ], - "Other": [ - "comment", - "text", - "other", - "size-in-bytes", - "counter", - "datetime", - "cpe", - "port", - "float", - "hex", - "phone-number", - "boolean" - ], - "Payload delivery": [ - "md5", - "sha1", - "sha224", - "sha256", - "sha384", - "sha512", - "sha512/224", - "sha512/256", - "ssdeep", - "imphash", - "impfuzzy", - "authentihash", - "pehash", - "tlsh", - "cdhash", - "filename", - "filename|md5", - "filename|sha1", - "filename|sha224", - "filename|sha256", - "filename|sha384", - "filename|sha512", - "filename|sha512/224", - "filename|sha512/256", - "filename|authentihash", - "filename|ssdeep", - "filename|tlsh", - "filename|imphash", - "filename|impfuzzy", - "filename|pehash", - "mac-address", - "mac-eui-64", - "ip-src", + "http-method", "ip-dst", "ip-dst|port", + "ip-src", "ip-src|port", - "hostname", - "domain", - "email-src", - "email-dst", - "email-subject", - "email-attachment", - "email-body", - "url", - "user-agent", - "AS", + "mac-address", + "mac-eui-64", + "other", "pattern-in-file", "pattern-in-traffic", + "port", + "snort", "stix2-pattern", - "yara", - "sigma", - "mime-type", - "attachment", - "malware-sample", - "link", - "malware-type", - "comment", "text", - "hex", - "vulnerability", - "x509-fingerprint-sha1", + "uri", + "url", + "user-agent", "x509-fingerprint-md5", - "x509-fingerprint-sha256", - "other", - "hostname|port", - "email-dst-display-name", - "email-src-display-name", - "email-header", - "email-reply-to", - "email-x-mailer", - "email-mime-boundary", - "email-thread-index", - "email-message-id", - "mobile-application-id", - "whois-registrant-email" + "x509-fingerprint-sha1", + "x509-fingerprint-sha256" ], - "Payload installation": [ - "md5", - "sha1", - "sha224", - "sha256", - "sha384", - "sha512", - "sha512/224", - "sha512/256", - "ssdeep", - "imphash", - "impfuzzy", + "Other": [ + "boolean", + "comment", + "counter", + "cpe", + "datetime", + "float", + "hex", + "other", + "phone-number", + "port", + "size-in-bytes", + "text" + ], + "Payload delivery": [ + "AS", + "attachment", "authentihash", - "pehash", - "tlsh", "cdhash", + "comment", + "domain", + "email-attachment", + "email-body", + "email-dst", + "email-dst-display-name", + "email-header", + "email-message-id", + "email-mime-boundary", + "email-reply-to", + "email-src", + "email-src-display-name", + "email-subject", + "email-thread-index", + "email-x-mailer", "filename", + "filename|authentihash", + "filename|impfuzzy", + "filename|imphash", "filename|md5", + "filename|pehash", "filename|sha1", "filename|sha224", "filename|sha256", @@ -312,105 +243,174 @@ "filename|sha512", "filename|sha512/224", "filename|sha512/256", - "filename|authentihash", "filename|ssdeep", "filename|tlsh", - "filename|imphash", - "filename|impfuzzy", - "filename|pehash", - "pattern-in-file", - "pattern-in-traffic", - "pattern-in-memory", - "stix2-pattern", - "yara", - "sigma", - "vulnerability", - "attachment", + "hex", + "hostname", + "hostname|port", + "impfuzzy", + "imphash", + "ip-dst", + "ip-dst|port", + "ip-src", + "ip-src|port", + "link", + "mac-address", + "mac-eui-64", "malware-sample", "malware-type", - "comment", - "text", - "hex", - "x509-fingerprint-sha1", - "x509-fingerprint-md5", - "x509-fingerprint-sha256", + "md5", + "mime-type", "mobile-application-id", "other", - "mime-type" + "pattern-in-file", + "pattern-in-traffic", + "pehash", + "sha1", + "sha224", + "sha256", + "sha384", + "sha512", + "sha512/224", + "sha512/256", + "sigma", + "ssdeep", + "stix2-pattern", + "text", + "tlsh", + "url", + "user-agent", + "vulnerability", + "whois-registrant-email", + "x509-fingerprint-md5", + "x509-fingerprint-sha1", + "x509-fingerprint-sha256", + "yara" + ], + "Payload installation": [ + "attachment", + "authentihash", + "cdhash", + "comment", + "filename", + "filename|authentihash", + "filename|impfuzzy", + "filename|imphash", + "filename|md5", + "filename|pehash", + "filename|sha1", + "filename|sha224", + "filename|sha256", + "filename|sha384", + "filename|sha512", + "filename|sha512/224", + "filename|sha512/256", + "filename|ssdeep", + "filename|tlsh", + "hex", + "impfuzzy", + "imphash", + "malware-sample", + "malware-type", + "md5", + "mime-type", + "mobile-application-id", + "other", + "pattern-in-file", + "pattern-in-memory", + "pattern-in-traffic", + "pehash", + "sha1", + "sha224", + "sha256", + "sha384", + "sha512", + "sha512/224", + "sha512/256", + "sigma", + "ssdeep", + "stix2-pattern", + "text", + "tlsh", + "vulnerability", + "x509-fingerprint-md5", + "x509-fingerprint-sha1", + "x509-fingerprint-sha256", + "yara" ], "Payload type": [ "comment", - "text", - "other" + "other", + "text" ], "Persistence mechanism": [ + "comment", "filename", + "hex", + "other", "regkey", "regkey|value", - "comment", - "text", - "other", - "hex" + "text" ], "Person": [ - "first-name", - "middle-name", - "last-name", + "comment", + "country-of-residence", "date-of-birth", - "place-of-birth", + "first-name", + "frequent-flyer-number", "gender", - "passport-number", + "identity-card-number", + "issue-date-of-the-visa", + "last-name", + "middle-name", + "nationality", + "other", + "passenger-name-record-locator-number", "passport-country", "passport-expiration", - "redress-number", - "nationality", - "visa-number", - "issue-date-of-the-visa", - "primary-residence", - "country-of-residence", - "special-service-request", - "frequent-flyer-number", - "travel-details", + "passport-number", "payment-details", - "place-port-of-original-embarkation", + "phone-number", + "place-of-birth", "place-port-of-clearance", "place-port-of-onward-foreign-destination", - "passenger-name-record-locator-number", - "comment", + "place-port-of-original-embarkation", + "primary-residence", + "redress-number", + "special-service-request", "text", - "other", - "phone-number", - "identity-card-number" + "travel-details", + "visa-number" ], "Social network": [ - "github-username", - "github-repository", - "github-organisation", - "jabber-id", - "twitter-id", - "email-src", - "email-dst", "comment", - "text", + "email-dst", + "email-src", + "github-organisation", + "github-repository", + "github-username", + "jabber-id", "other", + "text", + "twitter-id", "whois-registrant-email" ], "Support Tool": [ - "link", - "text", "attachment", "comment", + "hex", + "link", "other", - "hex" + "text" ], "Targeting data": [ - "target-user", + "comment", "target-email", + "target-external", + "target-location", "target-machine", "target-org", - "target-location", - "target-external", - "comment" + "target-user" ] }, "sane_defaults": { @@ -1028,159 +1028,159 @@ } }, "types": [ - "md5", - "sha1", - "sha256", - "filename", - "pdb", - "filename|md5", - "filename|sha1", - "filename|sha256", - "ip-src", - "ip-dst", - "hostname", + "AS", + "aba-rtn", + "attachment", + "authentihash", + "bank-account-nr", + "bic", + "bin", + "boolean", + "bro", + "btc", + "campaign-id", + "campaign-name", + "cc-number", + "cdhash", + "comment", + "cookie", + "cortex", + "counter", + "country-of-residence", + "cpe", + "date-of-birth", + "datetime", + "dns-soa-email", "domain", "domain|ip", - "email-src", - "email-dst", - "email-subject", "email-attachment", "email-body", - "float", - "url", - "http-method", - "user-agent", - "regkey", - "regkey|value", - "AS", - "snort", - "bro", - "pattern-in-file", - "pattern-in-traffic", - "pattern-in-memory", - "yara", - "stix2-pattern", - "sigma", - "gene", - "mime-type", - "identity-card-number", - "cookie", - "vulnerability", - "attachment", - "malware-sample", - "link", - "comment", - "text", - "hex", - "other", - "named pipe", - "mutex", - "target-user", - "target-email", - "target-machine", - "target-org", - "target-location", - "target-external", - "btc", - "xmr", - "iban", - "bic", - "bank-account-nr", - "aba-rtn", - "bin", - "cc-number", - "prtn", - "phone-number", - "threat-actor", - "campaign-name", - "campaign-id", - "malware-type", - "uri", - "authentihash", - "ssdeep", - "imphash", - "pehash", - "impfuzzy", - "sha224", - "sha384", - "sha512", - "sha512/224", - "sha512/256", - "tlsh", - "cdhash", + "email-dst", + "email-dst-display-name", + "email-header", + "email-message-id", + "email-mime-boundary", + "email-reply-to", + "email-src", + "email-src-display-name", + "email-subject", + "email-thread-index", + "email-x-mailer", + "filename", "filename|authentihash", - "filename|ssdeep", - "filename|imphash", "filename|impfuzzy", + "filename|imphash", + "filename|md5", "filename|pehash", + "filename|sha1", "filename|sha224", + "filename|sha256", "filename|sha384", "filename|sha512", "filename|sha512/224", "filename|sha512/256", + "filename|ssdeep", "filename|tlsh", - "windows-scheduled-task", - "windows-service-name", - "windows-service-displayname", - "whois-registrant-email", - "whois-registrant-phone", - "whois-registrant-name", - "whois-registrant-org", - "whois-registrar", - "whois-creation-date", - "x509-fingerprint-sha1", - "x509-fingerprint-md5", - "x509-fingerprint-sha256", - "dns-soa-email", - "size-in-bytes", - "counter", - "datetime", - "cpe", - "port", - "ip-dst|port", - "ip-src|port", + "first-name", + "float", + "frequent-flyer-number", + "gender", + "gene", + "github-organisation", + "github-repository", + "github-username", + "hex", + "hostname", "hostname|port", + "http-method", + "iban", + "identity-card-number", + "impfuzzy", + "imphash", + "ip-dst", + "ip-dst|port", + "ip-src", + "ip-src|port", + "issue-date-of-the-visa", + "jabber-id", + "last-name", + "link", "mac-address", "mac-eui-64", - "email-dst-display-name", - "email-src-display-name", - "email-header", - "email-reply-to", - "email-x-mailer", - "email-mime-boundary", - "email-thread-index", - "email-message-id", - "github-username", - "github-repository", - "github-organisation", - "jabber-id", - "twitter-id", - "first-name", + "malware-sample", + "malware-type", + "md5", "middle-name", - "last-name", - "date-of-birth", - "place-of-birth", - "gender", - "passport-number", + "mime-type", + "mobile-application-id", + "mutex", + "named pipe", + "nationality", + "other", + "passenger-name-record-locator-number", "passport-country", "passport-expiration", - "redress-number", - "nationality", - "visa-number", - "issue-date-of-the-visa", - "primary-residence", - "country-of-residence", - "special-service-request", - "frequent-flyer-number", - "travel-details", + "passport-number", + "pattern-in-file", + "pattern-in-memory", + "pattern-in-traffic", "payment-details", - "place-port-of-original-embarkation", + "pdb", + "pehash", + "phone-number", + "place-of-birth", "place-port-of-clearance", "place-port-of-onward-foreign-destination", - "passenger-name-record-locator-number", - "mobile-application-id", - "cortex", - "boolean" + "place-port-of-original-embarkation", + "port", + "primary-residence", + "prtn", + "redress-number", + "regkey", + "regkey|value", + "sha1", + "sha224", + "sha256", + "sha384", + "sha512", + "sha512/224", + "sha512/256", + "sigma", + "size-in-bytes", + "snort", + "special-service-request", + "ssdeep", + "stix2-pattern", + "target-email", + "target-external", + "target-location", + "target-machine", + "target-org", + "target-user", + "text", + "threat-actor", + "tlsh", + "travel-details", + "twitter-id", + "uri", + "url", + "user-agent", + "visa-number", + "vulnerability", + "whois-creation-date", + "whois-registrant-email", + "whois-registrant-name", + "whois-registrant-org", + "whois-registrant-phone", + "whois-registrar", + "windows-scheduled-task", + "windows-service-displayname", + "windows-service-name", + "x509-fingerprint-md5", + "x509-fingerprint-sha1", + "x509-fingerprint-sha256", + "xmr", + "yara" ] } } From 2f84b7ecdba9ae4204dbc09b38adf42c62a247ec Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 24 Dec 2018 20:27:19 +0100 Subject: [PATCH 25/38] chg: [data] describeTypes.json updated to the latest version --- pymisp/data/describeTypes.json | 2080 ++++++++++++++++---------------- 1 file changed, 1040 insertions(+), 1040 deletions(-) diff --git a/pymisp/data/describeTypes.json b/pymisp/data/describeTypes.json index 6db79e1..3b309ce 100644 --- a/pymisp/data/describeTypes.json +++ b/pymisp/data/describeTypes.json @@ -1,510 +1,49 @@ { "result": { - "categories": [ - "Antivirus detection", - "Artifacts dropped", - "Attribution", - "External analysis", - "Financial fraud", - "Internal reference", - "Network activity", - "Other", - "Payload delivery", - "Payload installation", - "Payload type", - "Persistence mechanism", - "Person", - "Social network", - "Support Tool", - "Targeting data" - ], - "category_type_mappings": { - "Antivirus detection": [ - "attachment", - "comment", - "hex", - "link", - "other", - "text" - ], - "Artifacts dropped": [ - "attachment", - "authentihash", - "cdhash", - "comment", - "cookie", - "filename", - "filename|authentihash", - "filename|impfuzzy", - "filename|imphash", - "filename|md5", - "filename|pehash", - "filename|sha1", - "filename|sha224", - "filename|sha256", - "filename|sha384", - "filename|sha512", - "filename|sha512/224", - "filename|sha512/256", - "filename|ssdeep", - "filename|tlsh", - "gene", - "hex", - "impfuzzy", - "imphash", - "malware-sample", - "md5", - "mime-type", - "mutex", - "named pipe", - "other", - "pattern-in-file", - "pattern-in-memory", - "pdb", - "regkey", - "regkey|value", - "sha1", - "sha224", - "sha256", - "sha384", - "sha512", - "sha512/224", - "sha512/256", - "sigma", - "ssdeep", - "stix2-pattern", - "text", - "windows-scheduled-task", - "windows-service-displayname", - "windows-service-name", - "x509-fingerprint-md5", - "x509-fingerprint-sha1", - "x509-fingerprint-sha256", - "yara" - ], - "Attribution": [ - "campaign-id", - "campaign-name", - "comment", - "dns-soa-email", - "other", - "text", - "threat-actor", - "whois-creation-date", - "whois-registrant-email", - "whois-registrant-name", - "whois-registrant-org", - "whois-registrant-phone", - "whois-registrar", - "x509-fingerprint-md5", - "x509-fingerprint-sha1", - "x509-fingerprint-sha256" - ], - "External analysis": [ - "AS", - "attachment", - "bro", - "comment", - "cortex", - "domain", - "domain|ip", - "filename", - "filename|md5", - "filename|sha1", - "filename|sha256", - "github-repository", - "hostname", - "ip-dst", - "ip-dst|port", - "ip-src", - "ip-src|port", - "link", - "mac-address", - "mac-eui-64", - "malware-sample", - "md5", - "other", - "pattern-in-file", - "pattern-in-memory", - "pattern-in-traffic", - "regkey", - "regkey|value", - "sha1", - "sha256", - "snort", - "text", - "url", - "user-agent", - "vulnerability", - "x509-fingerprint-md5", - "x509-fingerprint-sha1", - "x509-fingerprint-sha256" - ], - "Financial fraud": [ - "aba-rtn", - "bank-account-nr", - "bic", - "bin", - "btc", - "cc-number", - "comment", - "hex", - "iban", - "other", - "phone-number", - "prtn", - "text", - "xmr" - ], - "Internal reference": [ - "comment", - "hex", - "link", - "other", - "text" - ], - "Network activity": [ - "AS", - "attachment", - "bro", - "comment", - "cookie", - "domain", - "domain|ip", - "email-dst", - "hex", - "hostname", - "hostname|port", - "http-method", - "ip-dst", - "ip-dst|port", - "ip-src", - "ip-src|port", - "mac-address", - "mac-eui-64", - "other", - "pattern-in-file", - "pattern-in-traffic", - "port", - "snort", - "stix2-pattern", - "text", - "uri", - "url", - "user-agent", - "x509-fingerprint-md5", - "x509-fingerprint-sha1", - "x509-fingerprint-sha256" - ], - "Other": [ - "boolean", - "comment", - "counter", - "cpe", - "datetime", - "float", - "hex", - "other", - "phone-number", - "port", - "size-in-bytes", - "text" - ], - "Payload delivery": [ - "AS", - "attachment", - "authentihash", - "cdhash", - "comment", - "domain", - "email-attachment", - "email-body", - "email-dst", - "email-dst-display-name", - "email-header", - "email-message-id", - "email-mime-boundary", - "email-reply-to", - "email-src", - "email-src-display-name", - "email-subject", - "email-thread-index", - "email-x-mailer", - "filename", - "filename|authentihash", - "filename|impfuzzy", - "filename|imphash", - "filename|md5", - "filename|pehash", - "filename|sha1", - "filename|sha224", - "filename|sha256", - "filename|sha384", - "filename|sha512", - "filename|sha512/224", - "filename|sha512/256", - "filename|ssdeep", - "filename|tlsh", - "hex", - "hostname", - "hostname|port", - "impfuzzy", - "imphash", - "ip-dst", - "ip-dst|port", - "ip-src", - "ip-src|port", - "link", - "mac-address", - "mac-eui-64", - "malware-sample", - "malware-type", - "md5", - "mime-type", - "mobile-application-id", - "other", - "pattern-in-file", - "pattern-in-traffic", - "pehash", - "sha1", - "sha224", - "sha256", - "sha384", - "sha512", - "sha512/224", - "sha512/256", - "sigma", - "ssdeep", - "stix2-pattern", - "text", - "tlsh", - "url", - "user-agent", - "vulnerability", - "whois-registrant-email", - "x509-fingerprint-md5", - "x509-fingerprint-sha1", - "x509-fingerprint-sha256", - "yara" - ], - "Payload installation": [ - "attachment", - "authentihash", - "cdhash", - "comment", - "filename", - "filename|authentihash", - "filename|impfuzzy", - "filename|imphash", - "filename|md5", - "filename|pehash", - "filename|sha1", - "filename|sha224", - "filename|sha256", - "filename|sha384", - "filename|sha512", - "filename|sha512/224", - "filename|sha512/256", - "filename|ssdeep", - "filename|tlsh", - "hex", - "impfuzzy", - "imphash", - "malware-sample", - "malware-type", - "md5", - "mime-type", - "mobile-application-id", - "other", - "pattern-in-file", - "pattern-in-memory", - "pattern-in-traffic", - "pehash", - "sha1", - "sha224", - "sha256", - "sha384", - "sha512", - "sha512/224", - "sha512/256", - "sigma", - "ssdeep", - "stix2-pattern", - "text", - "tlsh", - "vulnerability", - "x509-fingerprint-md5", - "x509-fingerprint-sha1", - "x509-fingerprint-sha256", - "yara" - ], - "Payload type": [ - "comment", - "other", - "text" - ], - "Persistence mechanism": [ - "comment", - "filename", - "hex", - "other", - "regkey", - "regkey|value", - "text" - ], - "Person": [ - "comment", - "country-of-residence", - "date-of-birth", - "first-name", - "frequent-flyer-number", - "gender", - "identity-card-number", - "issue-date-of-the-visa", - "last-name", - "middle-name", - "nationality", - "other", - "passenger-name-record-locator-number", - "passport-country", - "passport-expiration", - "passport-number", - "payment-details", - "phone-number", - "place-of-birth", - "place-port-of-clearance", - "place-port-of-onward-foreign-destination", - "place-port-of-original-embarkation", - "primary-residence", - "redress-number", - "special-service-request", - "text", - "travel-details", - "visa-number" - ], - "Social network": [ - "comment", - "email-dst", - "email-src", - "github-organisation", - "github-repository", - "github-username", - "jabber-id", - "other", - "text", - "twitter-id", - "whois-registrant-email" - ], - "Support Tool": [ - "attachment", - "comment", - "hex", - "link", - "other", - "text" - ], - "Targeting data": [ - "comment", - "target-email", - "target-external", - "target-location", - "target-machine", - "target-org", - "target-user" - ] - }, "sane_defaults": { - "AS": { - "default_category": "Network activity", - "to_ids": 0 - }, - "aba-rtn": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "attachment": { - "default_category": "External analysis", - "to_ids": 0 - }, - "authentihash": { + "md5": { "default_category": "Payload delivery", "to_ids": 1 }, - "bank-account-nr": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "bic": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "bin": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "boolean": { - "default_category": "Other", - "to_ids": 0 - }, - "bro": { - "default_category": "Network activity", - "to_ids": 1 - }, - "btc": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "campaign-id": { - "default_category": "Attribution", - "to_ids": 0 - }, - "campaign-name": { - "default_category": "Attribution", - "to_ids": 0 - }, - "cc-number": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "cdhash": { + "sha1": { "default_category": "Payload delivery", "to_ids": 1 }, - "comment": { - "default_category": "Other", + "sha256": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "pdb": { + "default_category": "Artifacts dropped", "to_ids": 0 }, - "cookie": { + "filename|md5": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha1": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha256": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "ip-src": { "default_category": "Network activity", - "to_ids": 0 + "to_ids": 1 }, - "cortex": { - "default_category": "External analysis", - "to_ids": 0 + "ip-dst": { + "default_category": "Network activity", + "to_ids": 1 }, - "counter": { - "default_category": "Other", - "to_ids": 0 - }, - "country-of-residence": { - "default_category": "Person", - "to_ids": 0 - }, - "cpe": { - "default_category": "Other", - "to_ids": 0 - }, - "date-of-birth": { - "default_category": "Person", - "to_ids": 0 - }, - "datetime": { - "default_category": "Other", - "to_ids": 0 - }, - "dns-soa-email": { - "default_category": "Attribution", - "to_ids": 0 + "hostname": { + "default_category": "Network activity", + "to_ids": 1 }, "domain": { "default_category": "Network activity", @@ -514,6 +53,18 @@ "default_category": "Network activity", "to_ids": 1 }, + "email-src": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "email-dst": { + "default_category": "Network activity", + "to_ids": 1 + }, + "email-subject": { + "default_category": "Payload delivery", + "to_ids": 0 + }, "email-attachment": { "default_category": "Payload delivery", "to_ids": 1 @@ -522,151 +73,11 @@ "default_category": "Payload delivery", "to_ids": 0 }, - "email-dst": { - "default_category": "Network activity", - "to_ids": 1 - }, - "email-dst-display-name": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-header": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-message-id": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-mime-boundary": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-reply-to": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-src": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "email-src-display-name": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-subject": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-thread-index": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-x-mailer": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "filename": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|authentihash": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|impfuzzy": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|imphash": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|md5": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|pehash": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|sha1": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|sha224": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|sha256": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|sha384": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|sha512": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|sha512/224": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|sha512/256": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|ssdeep": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|tlsh": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "first-name": { - "default_category": "Person", - "to_ids": 0 - }, "float": { "default_category": "Other", "to_ids": 0 }, - "frequent-flyer-number": { - "default_category": "Person", - "to_ids": 0 - }, - "gender": { - "default_category": "Person", - "to_ids": 0 - }, - "gene": { - "default_category": "Artifacts dropped", - "to_ids": 0 - }, - "github-organisation": { - "default_category": "Social network", - "to_ids": 0 - }, - "github-repository": { - "default_category": "Social network", - "to_ids": 0 - }, - "github-username": { - "default_category": "Social network", - "to_ids": 0 - }, - "hex": { - "default_category": "Other", - "to_ids": 0 - }, - "hostname": { - "default_category": "Network activity", - "to_ids": 1 - }, - "hostname|port": { + "url": { "default_category": "Network activity", "to_ids": 1 }, @@ -674,176 +85,8 @@ "default_category": "Network activity", "to_ids": 0 }, - "iban": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "identity-card-number": { - "default_category": "Person", - "to_ids": 0 - }, - "impfuzzy": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "imphash": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "ip-dst": { + "user-agent": { "default_category": "Network activity", - "to_ids": 1 - }, - "ip-dst|port": { - "default_category": "Network activity", - "to_ids": 1 - }, - "ip-src": { - "default_category": "Network activity", - "to_ids": 1 - }, - "ip-src|port": { - "default_category": "Network activity", - "to_ids": 1 - }, - "issue-date-of-the-visa": { - "default_category": "Person", - "to_ids": 0 - }, - "jabber-id": { - "default_category": "Social network", - "to_ids": 0 - }, - "last-name": { - "default_category": "Person", - "to_ids": 0 - }, - "link": { - "default_category": "External analysis", - "to_ids": 0 - }, - "mac-address": { - "default_category": "Network activity", - "to_ids": 0 - }, - "mac-eui-64": { - "default_category": "Network activity", - "to_ids": 0 - }, - "malware-sample": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "malware-type": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "md5": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "middle-name": { - "default_category": "Person", - "to_ids": 0 - }, - "mime-type": { - "default_category": "Artifacts dropped", - "to_ids": 0 - }, - "mobile-application-id": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "mutex": { - "default_category": "Artifacts dropped", - "to_ids": 1 - }, - "named pipe": { - "default_category": "Artifacts dropped", - "to_ids": 0 - }, - "nationality": { - "default_category": "Person", - "to_ids": 0 - }, - "other": { - "default_category": "Other", - "to_ids": 0 - }, - "passenger-name-record-locator-number": { - "default_category": "Person", - "to_ids": 0 - }, - "passport-country": { - "default_category": "Person", - "to_ids": 0 - }, - "passport-expiration": { - "default_category": "Person", - "to_ids": 0 - }, - "passport-number": { - "default_category": "Person", - "to_ids": 0 - }, - "pattern-in-file": { - "default_category": "Payload installation", - "to_ids": 1 - }, - "pattern-in-memory": { - "default_category": "Payload installation", - "to_ids": 1 - }, - "pattern-in-traffic": { - "default_category": "Network activity", - "to_ids": 1 - }, - "payment-details": { - "default_category": "Person", - "to_ids": 0 - }, - "pdb": { - "default_category": "Artifacts dropped", - "to_ids": 0 - }, - "pehash": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "phone-number": { - "default_category": "Person", - "to_ids": 0 - }, - "place-of-birth": { - "default_category": "Person", - "to_ids": 0 - }, - "place-port-of-clearance": { - "default_category": "Person", - "to_ids": 0 - }, - "place-port-of-onward-foreign-destination": { - "default_category": "Person", - "to_ids": 0 - }, - "place-port-of-original-embarkation": { - "default_category": "Person", - "to_ids": 0 - }, - "port": { - "default_category": "Network activity", - "to_ids": 0 - }, - "primary-residence": { - "default_category": "Person", - "to_ids": 0 - }, - "prtn": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "redress-number": { - "default_category": "Person", "to_ids": 0 }, "regkey": { @@ -854,7 +97,199 @@ "default_category": "Persistence mechanism", "to_ids": 1 }, - "sha1": { + "AS": { + "default_category": "Network activity", + "to_ids": 0 + }, + "snort": { + "default_category": "Network activity", + "to_ids": 1 + }, + "bro": { + "default_category": "Network activity", + "to_ids": 1 + }, + "pattern-in-file": { + "default_category": "Payload installation", + "to_ids": 1 + }, + "pattern-in-traffic": { + "default_category": "Network activity", + "to_ids": 1 + }, + "pattern-in-memory": { + "default_category": "Payload installation", + "to_ids": 1 + }, + "yara": { + "default_category": "Payload installation", + "to_ids": 1 + }, + "stix2-pattern": { + "default_category": "Payload installation", + "to_ids": 1 + }, + "sigma": { + "default_category": "Payload installation", + "to_ids": 1 + }, + "gene": { + "default_category": "Artifacts dropped", + "to_ids": 0 + }, + "mime-type": { + "default_category": "Artifacts dropped", + "to_ids": 0 + }, + "identity-card-number": { + "default_category": "Person", + "to_ids": 0 + }, + "cookie": { + "default_category": "Network activity", + "to_ids": 0 + }, + "vulnerability": { + "default_category": "External analysis", + "to_ids": 0 + }, + "attachment": { + "default_category": "External analysis", + "to_ids": 0 + }, + "malware-sample": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "link": { + "default_category": "External analysis", + "to_ids": 0 + }, + "comment": { + "default_category": "Other", + "to_ids": 0 + }, + "text": { + "default_category": "Other", + "to_ids": 0 + }, + "hex": { + "default_category": "Other", + "to_ids": 0 + }, + "other": { + "default_category": "Other", + "to_ids": 0 + }, + "named pipe": { + "default_category": "Artifacts dropped", + "to_ids": 0 + }, + "mutex": { + "default_category": "Artifacts dropped", + "to_ids": 1 + }, + "target-user": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "target-email": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "target-machine": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "target-org": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "target-location": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "target-external": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "btc": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "xmr": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "iban": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "bic": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "bank-account-nr": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "aba-rtn": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "bin": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "cc-number": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "prtn": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "phone-number": { + "default_category": "Person", + "to_ids": 0 + }, + "threat-actor": { + "default_category": "Attribution", + "to_ids": 0 + }, + "campaign-name": { + "default_category": "Attribution", + "to_ids": 0 + }, + "campaign-id": { + "default_category": "Attribution", + "to_ids": 0 + }, + "malware-type": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "uri": { + "default_category": "Network activity", + "to_ids": 1 + }, + "authentihash": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "ssdeep": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "imphash": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "pehash": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "impfuzzy": { "default_category": "Payload delivery", "to_ids": 1 }, @@ -862,10 +297,6 @@ "default_category": "Payload delivery", "to_ids": 1 }, - "sha256": { - "default_category": "Payload delivery", - "to_ids": 1 - }, "sha384": { "default_category": "Payload delivery", "to_ids": 1 @@ -882,102 +313,78 @@ "default_category": "Payload delivery", "to_ids": 1 }, - "sigma": { - "default_category": "Payload installation", - "to_ids": 1 - }, - "size-in-bytes": { - "default_category": "Other", - "to_ids": 0 - }, - "snort": { - "default_category": "Network activity", - "to_ids": 1 - }, - "special-service-request": { - "default_category": "Person", - "to_ids": 0 - }, - "ssdeep": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "stix2-pattern": { - "default_category": "Payload installation", - "to_ids": 1 - }, - "target-email": { - "default_category": "Targeting data", - "to_ids": 0 - }, - "target-external": { - "default_category": "Targeting data", - "to_ids": 0 - }, - "target-location": { - "default_category": "Targeting data", - "to_ids": 0 - }, - "target-machine": { - "default_category": "Targeting data", - "to_ids": 0 - }, - "target-org": { - "default_category": "Targeting data", - "to_ids": 0 - }, - "target-user": { - "default_category": "Targeting data", - "to_ids": 0 - }, - "text": { - "default_category": "Other", - "to_ids": 0 - }, - "threat-actor": { - "default_category": "Attribution", - "to_ids": 0 - }, "tlsh": { "default_category": "Payload delivery", "to_ids": 1 }, - "travel-details": { - "default_category": "Person", - "to_ids": 0 - }, - "twitter-id": { - "default_category": "Social network", - "to_ids": 0 - }, - "uri": { - "default_category": "Network activity", + "cdhash": { + "default_category": "Payload delivery", "to_ids": 1 }, - "url": { - "default_category": "Network activity", + "filename|authentihash": { + "default_category": "Payload delivery", "to_ids": 1 }, - "user-agent": { - "default_category": "Network activity", + "filename|ssdeep": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|imphash": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|impfuzzy": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|pehash": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha224": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha384": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha512": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha512/224": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha512/256": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|tlsh": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "windows-scheduled-task": { + "default_category": "Artifacts dropped", "to_ids": 0 }, - "visa-number": { - "default_category": "Person", + "windows-service-name": { + "default_category": "Artifacts dropped", "to_ids": 0 }, - "vulnerability": { - "default_category": "External analysis", - "to_ids": 0 - }, - "whois-creation-date": { - "default_category": "Attribution", + "windows-service-displayname": { + "default_category": "Artifacts dropped", "to_ids": 0 }, "whois-registrant-email": { "default_category": "Attribution", "to_ids": 0 }, + "whois-registrant-phone": { + "default_category": "Attribution", + "to_ids": 0 + }, "whois-registrant-name": { "default_category": "Attribution", "to_ids": 0 @@ -986,31 +393,19 @@ "default_category": "Attribution", "to_ids": 0 }, - "whois-registrant-phone": { - "default_category": "Attribution", - "to_ids": 0 - }, "whois-registrar": { "default_category": "Attribution", "to_ids": 0 }, - "windows-scheduled-task": { - "default_category": "Artifacts dropped", + "whois-creation-date": { + "default_category": "Attribution", "to_ids": 0 }, - "windows-service-displayname": { - "default_category": "Artifacts dropped", - "to_ids": 0 - }, - "windows-service-name": { - "default_category": "Artifacts dropped", - "to_ids": 0 - }, - "x509-fingerprint-md5": { + "x509-fingerprint-sha1": { "default_category": "Network activity", "to_ids": 1 }, - "x509-fingerprint-sha1": { + "x509-fingerprint-md5": { "default_category": "Network activity", "to_ids": 1 }, @@ -1018,169 +413,774 @@ "default_category": "Network activity", "to_ids": 1 }, - "xmr": { - "default_category": "Financial fraud", + "dns-soa-email": { + "default_category": "Attribution", + "to_ids": 0 + }, + "size-in-bytes": { + "default_category": "Other", + "to_ids": 0 + }, + "counter": { + "default_category": "Other", + "to_ids": 0 + }, + "datetime": { + "default_category": "Other", + "to_ids": 0 + }, + "cpe": { + "default_category": "Other", + "to_ids": 0 + }, + "port": { + "default_category": "Network activity", + "to_ids": 0 + }, + "ip-dst|port": { + "default_category": "Network activity", "to_ids": 1 }, - "yara": { - "default_category": "Payload installation", + "ip-src|port": { + "default_category": "Network activity", "to_ids": 1 + }, + "hostname|port": { + "default_category": "Network activity", + "to_ids": 1 + }, + "mac-address": { + "default_category": "Network activity", + "to_ids": 0 + }, + "mac-eui-64": { + "default_category": "Network activity", + "to_ids": 0 + }, + "email-dst-display-name": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "email-src-display-name": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "email-header": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "email-reply-to": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "email-x-mailer": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "email-mime-boundary": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "email-thread-index": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "email-message-id": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "github-username": { + "default_category": "Social network", + "to_ids": 0 + }, + "github-repository": { + "default_category": "Social network", + "to_ids": 0 + }, + "github-organisation": { + "default_category": "Social network", + "to_ids": 0 + }, + "jabber-id": { + "default_category": "Social network", + "to_ids": 0 + }, + "twitter-id": { + "default_category": "Social network", + "to_ids": 0 + }, + "first-name": { + "default_category": "Person", + "to_ids": 0 + }, + "middle-name": { + "default_category": "Person", + "to_ids": 0 + }, + "last-name": { + "default_category": "Person", + "to_ids": 0 + }, + "date-of-birth": { + "default_category": "Person", + "to_ids": 0 + }, + "place-of-birth": { + "default_category": "Person", + "to_ids": 0 + }, + "gender": { + "default_category": "Person", + "to_ids": 0 + }, + "passport-number": { + "default_category": "Person", + "to_ids": 0 + }, + "passport-country": { + "default_category": "Person", + "to_ids": 0 + }, + "passport-expiration": { + "default_category": "Person", + "to_ids": 0 + }, + "redress-number": { + "default_category": "Person", + "to_ids": 0 + }, + "nationality": { + "default_category": "Person", + "to_ids": 0 + }, + "visa-number": { + "default_category": "Person", + "to_ids": 0 + }, + "issue-date-of-the-visa": { + "default_category": "Person", + "to_ids": 0 + }, + "primary-residence": { + "default_category": "Person", + "to_ids": 0 + }, + "country-of-residence": { + "default_category": "Person", + "to_ids": 0 + }, + "special-service-request": { + "default_category": "Person", + "to_ids": 0 + }, + "frequent-flyer-number": { + "default_category": "Person", + "to_ids": 0 + }, + "travel-details": { + "default_category": "Person", + "to_ids": 0 + }, + "payment-details": { + "default_category": "Person", + "to_ids": 0 + }, + "place-port-of-original-embarkation": { + "default_category": "Person", + "to_ids": 0 + }, + "place-port-of-clearance": { + "default_category": "Person", + "to_ids": 0 + }, + "place-port-of-onward-foreign-destination": { + "default_category": "Person", + "to_ids": 0 + }, + "passenger-name-record-locator-number": { + "default_category": "Person", + "to_ids": 0 + }, + "mobile-application-id": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "cortex": { + "default_category": "External analysis", + "to_ids": 0 + }, + "boolean": { + "default_category": "Other", + "to_ids": 0 } }, "types": [ - "AS", - "aba-rtn", - "attachment", - "authentihash", - "bank-account-nr", - "bic", - "bin", - "boolean", - "bro", - "btc", - "campaign-id", - "campaign-name", - "cc-number", - "cdhash", - "comment", - "cookie", - "cortex", - "counter", - "country-of-residence", - "cpe", - "date-of-birth", - "datetime", - "dns-soa-email", + "md5", + "sha1", + "sha256", + "filename", + "pdb", + "filename|md5", + "filename|sha1", + "filename|sha256", + "ip-src", + "ip-dst", + "hostname", "domain", "domain|ip", + "email-src", + "email-dst", + "email-subject", "email-attachment", "email-body", - "email-dst", - "email-dst-display-name", - "email-header", - "email-message-id", - "email-mime-boundary", - "email-reply-to", - "email-src", - "email-src-display-name", - "email-subject", - "email-thread-index", - "email-x-mailer", - "filename", - "filename|authentihash", - "filename|impfuzzy", - "filename|imphash", - "filename|md5", - "filename|pehash", - "filename|sha1", - "filename|sha224", - "filename|sha256", - "filename|sha384", - "filename|sha512", - "filename|sha512/224", - "filename|sha512/256", - "filename|ssdeep", - "filename|tlsh", - "first-name", "float", - "frequent-flyer-number", - "gender", - "gene", - "github-organisation", - "github-repository", - "github-username", - "hex", - "hostname", - "hostname|port", + "url", "http-method", - "iban", - "identity-card-number", - "impfuzzy", - "imphash", - "ip-dst", - "ip-dst|port", - "ip-src", - "ip-src|port", - "issue-date-of-the-visa", - "jabber-id", - "last-name", - "link", - "mac-address", - "mac-eui-64", - "malware-sample", - "malware-type", - "md5", - "middle-name", - "mime-type", - "mobile-application-id", - "mutex", - "named pipe", - "nationality", - "other", - "passenger-name-record-locator-number", - "passport-country", - "passport-expiration", - "passport-number", - "pattern-in-file", - "pattern-in-memory", - "pattern-in-traffic", - "payment-details", - "pdb", - "pehash", - "phone-number", - "place-of-birth", - "place-port-of-clearance", - "place-port-of-onward-foreign-destination", - "place-port-of-original-embarkation", - "port", - "primary-residence", - "prtn", - "redress-number", + "user-agent", "regkey", "regkey|value", - "sha1", + "AS", + "snort", + "bro", + "pattern-in-file", + "pattern-in-traffic", + "pattern-in-memory", + "yara", + "stix2-pattern", + "sigma", + "gene", + "mime-type", + "identity-card-number", + "cookie", + "vulnerability", + "attachment", + "malware-sample", + "link", + "comment", + "text", + "hex", + "other", + "named pipe", + "mutex", + "target-user", + "target-email", + "target-machine", + "target-org", + "target-location", + "target-external", + "btc", + "xmr", + "iban", + "bic", + "bank-account-nr", + "aba-rtn", + "bin", + "cc-number", + "prtn", + "phone-number", + "threat-actor", + "campaign-name", + "campaign-id", + "malware-type", + "uri", + "authentihash", + "ssdeep", + "imphash", + "pehash", + "impfuzzy", "sha224", - "sha256", "sha384", "sha512", "sha512/224", "sha512/256", - "sigma", - "size-in-bytes", - "snort", - "special-service-request", - "ssdeep", - "stix2-pattern", - "target-email", - "target-external", - "target-location", - "target-machine", - "target-org", - "target-user", - "text", - "threat-actor", "tlsh", - "travel-details", - "twitter-id", - "uri", - "url", - "user-agent", - "visa-number", - "vulnerability", - "whois-creation-date", + "cdhash", + "filename|authentihash", + "filename|ssdeep", + "filename|imphash", + "filename|impfuzzy", + "filename|pehash", + "filename|sha224", + "filename|sha384", + "filename|sha512", + "filename|sha512/224", + "filename|sha512/256", + "filename|tlsh", + "windows-scheduled-task", + "windows-service-name", + "windows-service-displayname", "whois-registrant-email", + "whois-registrant-phone", "whois-registrant-name", "whois-registrant-org", - "whois-registrant-phone", "whois-registrar", - "windows-scheduled-task", - "windows-service-displayname", - "windows-service-name", - "x509-fingerprint-md5", + "whois-creation-date", "x509-fingerprint-sha1", + "x509-fingerprint-md5", "x509-fingerprint-sha256", - "xmr", - "yara" - ] + "dns-soa-email", + "size-in-bytes", + "counter", + "datetime", + "cpe", + "port", + "ip-dst|port", + "ip-src|port", + "hostname|port", + "mac-address", + "mac-eui-64", + "email-dst-display-name", + "email-src-display-name", + "email-header", + "email-reply-to", + "email-x-mailer", + "email-mime-boundary", + "email-thread-index", + "email-message-id", + "github-username", + "github-repository", + "github-organisation", + "jabber-id", + "twitter-id", + "first-name", + "middle-name", + "last-name", + "date-of-birth", + "place-of-birth", + "gender", + "passport-number", + "passport-country", + "passport-expiration", + "redress-number", + "nationality", + "visa-number", + "issue-date-of-the-visa", + "primary-residence", + "country-of-residence", + "special-service-request", + "frequent-flyer-number", + "travel-details", + "payment-details", + "place-port-of-original-embarkation", + "place-port-of-clearance", + "place-port-of-onward-foreign-destination", + "passenger-name-record-locator-number", + "mobile-application-id", + "cortex", + "boolean" + ], + "categories": [ + "Internal reference", + "Targeting data", + "Antivirus detection", + "Payload delivery", + "Artifacts dropped", + "Payload installation", + "Persistence mechanism", + "Network activity", + "Payload type", + "Attribution", + "External analysis", + "Financial fraud", + "Support Tool", + "Social network", + "Person", + "Other" + ], + "category_type_mappings": { + "Internal reference": [ + "text", + "link", + "comment", + "other", + "hex" + ], + "Targeting data": [ + "target-user", + "target-email", + "target-machine", + "target-org", + "target-location", + "target-external", + "comment" + ], + "Antivirus detection": [ + "link", + "comment", + "text", + "hex", + "attachment", + "other" + ], + "Payload delivery": [ + "md5", + "sha1", + "sha224", + "sha256", + "sha384", + "sha512", + "sha512/224", + "sha512/256", + "ssdeep", + "imphash", + "impfuzzy", + "authentihash", + "pehash", + "tlsh", + "cdhash", + "filename", + "filename|md5", + "filename|sha1", + "filename|sha224", + "filename|sha256", + "filename|sha384", + "filename|sha512", + "filename|sha512/224", + "filename|sha512/256", + "filename|authentihash", + "filename|ssdeep", + "filename|tlsh", + "filename|imphash", + "filename|impfuzzy", + "filename|pehash", + "mac-address", + "mac-eui-64", + "ip-src", + "ip-dst", + "ip-dst|port", + "ip-src|port", + "hostname", + "domain", + "email-src", + "email-dst", + "email-subject", + "email-attachment", + "email-body", + "url", + "user-agent", + "AS", + "pattern-in-file", + "pattern-in-traffic", + "stix2-pattern", + "yara", + "sigma", + "mime-type", + "attachment", + "malware-sample", + "link", + "malware-type", + "comment", + "text", + "hex", + "vulnerability", + "x509-fingerprint-sha1", + "x509-fingerprint-md5", + "x509-fingerprint-sha256", + "other", + "hostname|port", + "email-dst-display-name", + "email-src-display-name", + "email-header", + "email-reply-to", + "email-x-mailer", + "email-mime-boundary", + "email-thread-index", + "email-message-id", + "mobile-application-id", + "whois-registrant-email" + ], + "Artifacts dropped": [ + "md5", + "sha1", + "sha224", + "sha256", + "sha384", + "sha512", + "sha512/224", + "sha512/256", + "ssdeep", + "imphash", + "impfuzzy", + "authentihash", + "cdhash", + "filename", + "filename|md5", + "filename|sha1", + "filename|sha224", + "filename|sha256", + "filename|sha384", + "filename|sha512", + "filename|sha512/224", + "filename|sha512/256", + "filename|authentihash", + "filename|ssdeep", + "filename|tlsh", + "filename|imphash", + "filename|impfuzzy", + "filename|pehash", + "regkey", + "regkey|value", + "pattern-in-file", + "pattern-in-memory", + "pdb", + "stix2-pattern", + "yara", + "sigma", + "attachment", + "malware-sample", + "named pipe", + "mutex", + "windows-scheduled-task", + "windows-service-name", + "windows-service-displayname", + "comment", + "text", + "hex", + "x509-fingerprint-sha1", + "x509-fingerprint-md5", + "x509-fingerprint-sha256", + "other", + "cookie", + "gene", + "mime-type" + ], + "Payload installation": [ + "md5", + "sha1", + "sha224", + "sha256", + "sha384", + "sha512", + "sha512/224", + "sha512/256", + "ssdeep", + "imphash", + "impfuzzy", + "authentihash", + "pehash", + "tlsh", + "cdhash", + "filename", + "filename|md5", + "filename|sha1", + "filename|sha224", + "filename|sha256", + "filename|sha384", + "filename|sha512", + "filename|sha512/224", + "filename|sha512/256", + "filename|authentihash", + "filename|ssdeep", + "filename|tlsh", + "filename|imphash", + "filename|impfuzzy", + "filename|pehash", + "pattern-in-file", + "pattern-in-traffic", + "pattern-in-memory", + "stix2-pattern", + "yara", + "sigma", + "vulnerability", + "attachment", + "malware-sample", + "malware-type", + "comment", + "text", + "hex", + "x509-fingerprint-sha1", + "x509-fingerprint-md5", + "x509-fingerprint-sha256", + "mobile-application-id", + "other", + "mime-type" + ], + "Persistence mechanism": [ + "filename", + "regkey", + "regkey|value", + "comment", + "text", + "other", + "hex" + ], + "Network activity": [ + "ip-src", + "ip-dst", + "ip-dst|port", + "ip-src|port", + "port", + "hostname", + "domain", + "domain|ip", + "mac-address", + "mac-eui-64", + "email-dst", + "url", + "uri", + "user-agent", + "http-method", + "AS", + "snort", + "pattern-in-file", + "stix2-pattern", + "pattern-in-traffic", + "attachment", + "comment", + "text", + "x509-fingerprint-md5", + "x509-fingerprint-sha1", + "x509-fingerprint-sha256", + "other", + "hex", + "cookie", + "hostname|port", + "bro" + ], + "Payload type": [ + "comment", + "text", + "other" + ], + "Attribution": [ + "threat-actor", + "campaign-name", + "campaign-id", + "whois-registrant-phone", + "whois-registrant-email", + "whois-registrant-name", + "whois-registrant-org", + "whois-registrar", + "whois-creation-date", + "comment", + "text", + "x509-fingerprint-sha1", + "x509-fingerprint-md5", + "x509-fingerprint-sha256", + "other", + "dns-soa-email" + ], + "External analysis": [ + "md5", + "sha1", + "sha256", + "filename", + "filename|md5", + "filename|sha1", + "filename|sha256", + "ip-src", + "ip-dst", + "ip-dst|port", + "ip-src|port", + "mac-address", + "mac-eui-64", + "hostname", + "domain", + "domain|ip", + "url", + "user-agent", + "regkey", + "regkey|value", + "AS", + "snort", + "bro", + "pattern-in-file", + "pattern-in-traffic", + "pattern-in-memory", + "vulnerability", + "attachment", + "malware-sample", + "link", + "comment", + "text", + "x509-fingerprint-sha1", + "x509-fingerprint-md5", + "x509-fingerprint-sha256", + "github-repository", + "other", + "cortex" + ], + "Financial fraud": [ + "btc", + "xmr", + "iban", + "bic", + "bank-account-nr", + "aba-rtn", + "bin", + "cc-number", + "prtn", + "phone-number", + "comment", + "text", + "other", + "hex" + ], + "Support Tool": [ + "link", + "text", + "attachment", + "comment", + "other", + "hex" + ], + "Social network": [ + "github-username", + "github-repository", + "github-organisation", + "jabber-id", + "twitter-id", + "email-src", + "email-dst", + "comment", + "text", + "other", + "whois-registrant-email" + ], + "Person": [ + "first-name", + "middle-name", + "last-name", + "date-of-birth", + "place-of-birth", + "gender", + "passport-number", + "passport-country", + "passport-expiration", + "redress-number", + "nationality", + "visa-number", + "issue-date-of-the-visa", + "primary-residence", + "country-of-residence", + "special-service-request", + "frequent-flyer-number", + "travel-details", + "payment-details", + "place-port-of-original-embarkation", + "place-port-of-clearance", + "place-port-of-onward-foreign-destination", + "passenger-name-record-locator-number", + "comment", + "text", + "other", + "phone-number", + "identity-card-number" + ], + "Other": [ + "comment", + "text", + "other", + "size-in-bytes", + "counter", + "datetime", + "cpe", + "port", + "float", + "hex", + "phone-number", + "boolean" + ] + } } } From 10ccd637d9b3ece6703bdcfa735177fb5cb71ed3 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 24 Dec 2018 20:46:26 +0100 Subject: [PATCH 26/38] chg: [test] set a default distribution for massive event creation --- examples/events/tools.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/events/tools.py b/examples/events/tools.py index 9d0e3f5..76b70a2 100644 --- a/examples/events/tools.py +++ b/examples/events/tools.py @@ -65,10 +65,11 @@ def create_dummy_event(misp): def create_massive_dummy_events(misp, nbattribute): event = misp.new_event(0, 4, 0, 'massive dummy event') eventid = event['Event']['id'] + distribution = '0' functions = [floodtxt, floodip, flooddomain, flooddomainip, floodemail, floodattachment] for i in range(nbattribute): choice = randint(0, 5) if choice == 5: - floodattachment(misp, eventid, event['Event']['distribution'], False, 'Payload delivery', '', event['Event']['info'], event['Event']['analysis'], event['Event']['threat_level_id']) + floodattachment(misp, eventid, distribution, False, 'Payload delivery', '', event['Event']['info'], event['Event']['analysis'], event['Event']['threat_level_id']) else: functions[choice](misp, event) From a3108f76896111e39b0e4e2688c3652ba29f566d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 26 Dec 2018 17:38:19 +0100 Subject: [PATCH 27/38] fix: error vs errors key --- pymisp/aping.py | 4 ++-- tests/testlive_comprehensive.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pymisp/aping.py b/pymisp/aping.py index 88d8a6b..097b8f9 100644 --- a/pymisp/aping.py +++ b/pymisp/aping.py @@ -69,7 +69,7 @@ class ExpandedPyMISP(PyMISP): # The server returns a json message with the error details error_message = response.json() logger.error(f'Something went wrong ({response.status_code}): {error_message}') - return {'error': (response.status_code, error_message)} + return {'errors': (response.status_code, error_message)} # At this point, we had no error. @@ -87,7 +87,7 @@ class ExpandedPyMISP(PyMISP): if not len(response.content): # Empty response logger.error('Got an empty response.') - return {'error': 'The response is empty.'} + return {'errors': 'The response is empty.'} return response.text def get_event(self, event_id: int): diff --git a/tests/testlive_comprehensive.py b/tests/testlive_comprehensive.py index d7837ea..09ceb58 100644 --- a/tests/testlive_comprehensive.py +++ b/tests/testlive_comprehensive.py @@ -755,7 +755,7 @@ class TestComprehensive(unittest.TestCase): second = self.user_misp_connector.add_event(second) response = self.user_misp_connector.fast_publish(first.id, alert=False) - self.assertEqual(response['errors'][0][1]['message'], 'You do not have permission to use this functionality.') + self.assertEqual(response['errors'][1]['message'], 'You do not have permission to use this functionality.') # Default search, attribute with to_ids == True first.attributes[0].to_ids = True From 4c9e6d0ec8502f19e01ed4efceca60642ae67d86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 26 Dec 2018 18:27:48 +0100 Subject: [PATCH 28/38] fix: Create massive event using ExpandedPyMISP --- examples/events/create_massive_dummy_events.py | 10 +++++++--- examples/events/tools.py | 10 +++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/examples/events/create_massive_dummy_events.py b/examples/events/create_massive_dummy_events.py index 12a2826..7829ad6 100755 --- a/examples/events/create_massive_dummy_events.py +++ b/examples/events/create_massive_dummy_events.py @@ -1,8 +1,12 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from pymisp import PyMISP -from keys import url, key +from pymisp import ExpandedPyMISP +try: + from keys import url, key +except ImportError: + url = 'http://localhost:8080' + key = '8h0gHbhS0fv6JUOlTED0AznLXFbf83TYtQrCycqb' import argparse import tools @@ -13,7 +17,7 @@ if __name__ == '__main__': parser.add_argument("-a", "--attribute", type=int, help="Number of attributes per event (default 3000)") args = parser.parse_args() - misp = PyMISP(url, key, True, 'json') + misp = ExpandedPyMISP(url, key, True) if args.limit is None: args.limit = 1 diff --git a/examples/events/tools.py b/examples/events/tools.py index 76b70a2..94f5d91 100644 --- a/examples/events/tools.py +++ b/examples/events/tools.py @@ -4,6 +4,7 @@ import random from random import randint import string +from pymisp import MISPEvent def randomStringGenerator(size, chars=string.ascii_lowercase + string.digits): @@ -63,13 +64,16 @@ def create_dummy_event(misp): def create_massive_dummy_events(misp, nbattribute): - event = misp.new_event(0, 4, 0, 'massive dummy event') - eventid = event['Event']['id'] + event = MISPEvent() + event.info = 'massive dummy event' + event = misp.add_event(event) + print(event) + eventid = event.id distribution = '0' functions = [floodtxt, floodip, flooddomain, flooddomainip, floodemail, floodattachment] for i in range(nbattribute): choice = randint(0, 5) if choice == 5: - floodattachment(misp, eventid, distribution, False, 'Payload delivery', '', event['Event']['info'], event['Event']['analysis'], event['Event']['threat_level_id']) + floodattachment(misp, eventid, distribution, False, 'Payload delivery', '', event.info, event.analysis, event.threat_level_id) else: functions[choice](misp, event) From 1c6b3d82456de836602dbda21ea0626c92c14c72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 26 Dec 2018 19:22:47 +0100 Subject: [PATCH 29/38] new: Add test for references when adding/updating a full event --- tests/testlive_comprehensive.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/testlive_comprehensive.py b/tests/testlive_comprehensive.py index 09ceb58..ab097c2 100644 --- a/tests/testlive_comprehensive.py +++ b/tests/testlive_comprehensive.py @@ -4,6 +4,7 @@ import unittest from pymisp import ExpandedPyMISP, MISPEvent, MISPOrganisation, MISPUser, Distribution, ThreatLevel, Analysis, MISPObject +from pymisp.tools import make_binary_objects from datetime import datetime, timedelta, date from io import BytesIO @@ -881,6 +882,24 @@ class TestComprehensive(unittest.TestCase): self.admin_misp_connector.enable_tag(tag['id']) # FIXME: returns the tag with ID 1 + def test_add_event_with_attachment(self): + first = self.create_simple_event() + try: + first = self.user_misp_connector.add_event(first) + file_obj, bin_obj, sections = make_binary_objects('tests/viper-test-files/test_files/whoami.exe', 'rb') + first.add_object(file_obj) + first.add_object(bin_obj) + for s in sections: + first.add_object(s) + self.assertEqual(len(first.objects[0].references), 1) + self.assertEqual(first.objects[0].references[0].relationship_type, 'included-in') + first = self.user_misp_connector.update_event(first) + self.assertEqual(len(first.objects[0].references), 1) + self.assertEqual(first.objects[0].references[0].relationship_type, 'included-in') + finally: + # Delete event + self.admin_misp_connector.delete_event(first.id) + def test_taxonomies(self): # Make sure we're up-to-date self.admin_misp_connector.update_taxonomies() From 7bbcdf262baf0c21608dcadc4f143efe55e0ecd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 26 Dec 2018 19:41:59 +0100 Subject: [PATCH 30/38] fix: Test case was broken --- tests/testlive_comprehensive.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/testlive_comprehensive.py b/tests/testlive_comprehensive.py index ab097c2..a8d147b 100644 --- a/tests/testlive_comprehensive.py +++ b/tests/testlive_comprehensive.py @@ -18,7 +18,7 @@ try: except ImportError as e: print(e) url = 'http://localhost:8080' - key = 'LBelWqKY9SQyG0huZzAMqiEBl6FODxpgRRXMsZFu' + key = '8h0gHbhS0fv6JUOlTED0AznLXFbf83TYtQrCycqb' verifycert = False travis_run = False @@ -38,7 +38,7 @@ class TestComprehensive(unittest.TestCase): usr = cls.admin_misp_connector.add_user(email='testusr@user.local', org_id=cls.test_org.id, role_id=3) cls.test_usr = MISPUser() cls.test_usr.from_dict(**usr) - cls.user_misp_connector = ExpandedPyMISP(url, cls.test_usr.authkey, verifycert) + cls.user_misp_connector = ExpandedPyMISP(url, cls.test_usr.authkey, verifycert, debug=False) # Creates a publisher pub = cls.admin_misp_connector.add_user(email='testpub@user.local', org_id=cls.test_org.id, role_id=4) cls.test_pub = MISPUser() @@ -886,7 +886,7 @@ class TestComprehensive(unittest.TestCase): first = self.create_simple_event() try: first = self.user_misp_connector.add_event(first) - file_obj, bin_obj, sections = make_binary_objects('tests/viper-test-files/test_files/whoami.exe', 'rb') + file_obj, bin_obj, sections = make_binary_objects('tests/viper-test-files/test_files/whoami.exe', standalone=False) first.add_object(file_obj) first.add_object(bin_obj) for s in sections: From 444f9a6fd9d78e04797288455d9ef9f8618ccb5f Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Sun, 30 Dec 2018 12:49:44 +0100 Subject: [PATCH 31/38] chg: [data] ja3-fingerprint-md5 type added --- pymisp/data/describeTypes.json | 1934 ++++++++++++++++---------------- 1 file changed, 971 insertions(+), 963 deletions(-) diff --git a/pymisp/data/describeTypes.json b/pymisp/data/describeTypes.json index 3b309ce..850668d 100644 --- a/pymisp/data/describeTypes.json +++ b/pymisp/data/describeTypes.json @@ -1,49 +1,513 @@ { "result": { + "categories": [ + "Antivirus detection", + "Artifacts dropped", + "Attribution", + "External analysis", + "Financial fraud", + "Internal reference", + "Network activity", + "Other", + "Payload delivery", + "Payload installation", + "Payload type", + "Persistence mechanism", + "Person", + "Social network", + "Support Tool", + "Targeting data" + ], + "category_type_mappings": { + "Antivirus detection": [ + "attachment", + "comment", + "hex", + "link", + "other", + "text" + ], + "Artifacts dropped": [ + "attachment", + "authentihash", + "cdhash", + "comment", + "cookie", + "filename", + "filename|authentihash", + "filename|impfuzzy", + "filename|imphash", + "filename|md5", + "filename|pehash", + "filename|sha1", + "filename|sha224", + "filename|sha256", + "filename|sha384", + "filename|sha512", + "filename|sha512/224", + "filename|sha512/256", + "filename|ssdeep", + "filename|tlsh", + "gene", + "hex", + "impfuzzy", + "imphash", + "malware-sample", + "md5", + "mime-type", + "mutex", + "named pipe", + "other", + "pattern-in-file", + "pattern-in-memory", + "pdb", + "regkey", + "regkey|value", + "sha1", + "sha224", + "sha256", + "sha384", + "sha512", + "sha512/224", + "sha512/256", + "sigma", + "ssdeep", + "stix2-pattern", + "text", + "windows-scheduled-task", + "windows-service-displayname", + "windows-service-name", + "x509-fingerprint-md5", + "x509-fingerprint-sha1", + "x509-fingerprint-sha256", + "yara" + ], + "Attribution": [ + "campaign-id", + "campaign-name", + "comment", + "dns-soa-email", + "other", + "text", + "threat-actor", + "whois-creation-date", + "whois-registrant-email", + "whois-registrant-name", + "whois-registrant-org", + "whois-registrant-phone", + "whois-registrar", + "x509-fingerprint-md5", + "x509-fingerprint-sha1", + "x509-fingerprint-sha256" + ], + "External analysis": [ + "AS", + "attachment", + "bro", + "comment", + "cortex", + "domain", + "domain|ip", + "filename", + "filename|md5", + "filename|sha1", + "filename|sha256", + "github-repository", + "hostname", + "ip-dst", + "ip-dst|port", + "ip-src", + "ip-src|port", + "ja3-fingerprint-md5", + "link", + "mac-address", + "mac-eui-64", + "malware-sample", + "md5", + "other", + "pattern-in-file", + "pattern-in-memory", + "pattern-in-traffic", + "regkey", + "regkey|value", + "sha1", + "sha256", + "snort", + "text", + "url", + "user-agent", + "vulnerability", + "x509-fingerprint-md5", + "x509-fingerprint-sha1", + "x509-fingerprint-sha256" + ], + "Financial fraud": [ + "aba-rtn", + "bank-account-nr", + "bic", + "bin", + "btc", + "cc-number", + "comment", + "hex", + "iban", + "other", + "phone-number", + "prtn", + "text", + "xmr" + ], + "Internal reference": [ + "comment", + "hex", + "link", + "other", + "text" + ], + "Network activity": [ + "AS", + "attachment", + "bro", + "comment", + "cookie", + "domain", + "domain|ip", + "email-dst", + "hex", + "hostname", + "hostname|port", + "http-method", + "ip-dst", + "ip-dst|port", + "ip-src", + "ip-src|port", + "ja3-fingerprint-md5", + "mac-address", + "mac-eui-64", + "other", + "pattern-in-file", + "pattern-in-traffic", + "port", + "snort", + "stix2-pattern", + "text", + "uri", + "url", + "user-agent", + "x509-fingerprint-md5", + "x509-fingerprint-sha1", + "x509-fingerprint-sha256" + ], + "Other": [ + "boolean", + "comment", + "counter", + "cpe", + "datetime", + "float", + "hex", + "other", + "phone-number", + "port", + "size-in-bytes", + "text" + ], + "Payload delivery": [ + "AS", + "attachment", + "authentihash", + "cdhash", + "comment", + "domain", + "email-attachment", + "email-body", + "email-dst", + "email-dst-display-name", + "email-header", + "email-message-id", + "email-mime-boundary", + "email-reply-to", + "email-src", + "email-src-display-name", + "email-subject", + "email-thread-index", + "email-x-mailer", + "filename", + "filename|authentihash", + "filename|impfuzzy", + "filename|imphash", + "filename|md5", + "filename|pehash", + "filename|sha1", + "filename|sha224", + "filename|sha256", + "filename|sha384", + "filename|sha512", + "filename|sha512/224", + "filename|sha512/256", + "filename|ssdeep", + "filename|tlsh", + "hex", + "hostname", + "hostname|port", + "impfuzzy", + "imphash", + "ip-dst", + "ip-dst|port", + "ip-src", + "ip-src|port", + "ja3-fingerprint-md5", + "link", + "mac-address", + "mac-eui-64", + "malware-sample", + "malware-type", + "md5", + "mime-type", + "mobile-application-id", + "other", + "pattern-in-file", + "pattern-in-traffic", + "pehash", + "sha1", + "sha224", + "sha256", + "sha384", + "sha512", + "sha512/224", + "sha512/256", + "sigma", + "ssdeep", + "stix2-pattern", + "text", + "tlsh", + "url", + "user-agent", + "vulnerability", + "whois-registrant-email", + "x509-fingerprint-md5", + "x509-fingerprint-sha1", + "x509-fingerprint-sha256", + "yara" + ], + "Payload installation": [ + "attachment", + "authentihash", + "cdhash", + "comment", + "filename", + "filename|authentihash", + "filename|impfuzzy", + "filename|imphash", + "filename|md5", + "filename|pehash", + "filename|sha1", + "filename|sha224", + "filename|sha256", + "filename|sha384", + "filename|sha512", + "filename|sha512/224", + "filename|sha512/256", + "filename|ssdeep", + "filename|tlsh", + "hex", + "impfuzzy", + "imphash", + "malware-sample", + "malware-type", + "md5", + "mime-type", + "mobile-application-id", + "other", + "pattern-in-file", + "pattern-in-memory", + "pattern-in-traffic", + "pehash", + "sha1", + "sha224", + "sha256", + "sha384", + "sha512", + "sha512/224", + "sha512/256", + "sigma", + "ssdeep", + "stix2-pattern", + "text", + "tlsh", + "vulnerability", + "x509-fingerprint-md5", + "x509-fingerprint-sha1", + "x509-fingerprint-sha256", + "yara" + ], + "Payload type": [ + "comment", + "other", + "text" + ], + "Persistence mechanism": [ + "comment", + "filename", + "hex", + "other", + "regkey", + "regkey|value", + "text" + ], + "Person": [ + "comment", + "country-of-residence", + "date-of-birth", + "first-name", + "frequent-flyer-number", + "gender", + "identity-card-number", + "issue-date-of-the-visa", + "last-name", + "middle-name", + "nationality", + "other", + "passenger-name-record-locator-number", + "passport-country", + "passport-expiration", + "passport-number", + "payment-details", + "phone-number", + "place-of-birth", + "place-port-of-clearance", + "place-port-of-onward-foreign-destination", + "place-port-of-original-embarkation", + "primary-residence", + "redress-number", + "special-service-request", + "text", + "travel-details", + "visa-number" + ], + "Social network": [ + "comment", + "email-dst", + "email-src", + "github-organisation", + "github-repository", + "github-username", + "jabber-id", + "other", + "text", + "twitter-id", + "whois-registrant-email" + ], + "Support Tool": [ + "attachment", + "comment", + "hex", + "link", + "other", + "text" + ], + "Targeting data": [ + "comment", + "target-email", + "target-external", + "target-location", + "target-machine", + "target-org", + "target-user" + ] + }, "sane_defaults": { - "md5": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "sha1": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "sha256": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "pdb": { - "default_category": "Artifacts dropped", + "AS": { + "default_category": "Network activity", "to_ids": 0 }, - "filename|md5": { + "aba-rtn": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "attachment": { + "default_category": "External analysis", + "to_ids": 0 + }, + "authentihash": { "default_category": "Payload delivery", "to_ids": 1 }, - "filename|sha1": { + "bank-account-nr": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "bic": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "bin": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "boolean": { + "default_category": "Other", + "to_ids": 0 + }, + "bro": { + "default_category": "Network activity", + "to_ids": 1 + }, + "btc": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "campaign-id": { + "default_category": "Attribution", + "to_ids": 0 + }, + "campaign-name": { + "default_category": "Attribution", + "to_ids": 0 + }, + "cc-number": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "cdhash": { "default_category": "Payload delivery", "to_ids": 1 }, - "filename|sha256": { - "default_category": "Payload delivery", - "to_ids": 1 + "comment": { + "default_category": "Other", + "to_ids": 0 }, - "ip-src": { + "cookie": { "default_category": "Network activity", - "to_ids": 1 + "to_ids": 0 }, - "ip-dst": { - "default_category": "Network activity", - "to_ids": 1 + "cortex": { + "default_category": "External analysis", + "to_ids": 0 }, - "hostname": { - "default_category": "Network activity", - "to_ids": 1 + "counter": { + "default_category": "Other", + "to_ids": 0 + }, + "country-of-residence": { + "default_category": "Person", + "to_ids": 0 + }, + "cpe": { + "default_category": "Other", + "to_ids": 0 + }, + "date-of-birth": { + "default_category": "Person", + "to_ids": 0 + }, + "datetime": { + "default_category": "Other", + "to_ids": 0 + }, + "dns-soa-email": { + "default_category": "Attribution", + "to_ids": 0 }, "domain": { "default_category": "Network activity", @@ -53,18 +517,6 @@ "default_category": "Network activity", "to_ids": 1 }, - "email-src": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "email-dst": { - "default_category": "Network activity", - "to_ids": 1 - }, - "email-subject": { - "default_category": "Payload delivery", - "to_ids": 0 - }, "email-attachment": { "default_category": "Payload delivery", "to_ids": 1 @@ -73,251 +525,51 @@ "default_category": "Payload delivery", "to_ids": 0 }, - "float": { - "default_category": "Other", - "to_ids": 0 - }, - "url": { + "email-dst": { "default_category": "Network activity", "to_ids": 1 }, - "http-method": { - "default_category": "Network activity", - "to_ids": 0 - }, - "user-agent": { - "default_category": "Network activity", - "to_ids": 0 - }, - "regkey": { - "default_category": "Persistence mechanism", - "to_ids": 1 - }, - "regkey|value": { - "default_category": "Persistence mechanism", - "to_ids": 1 - }, - "AS": { - "default_category": "Network activity", - "to_ids": 0 - }, - "snort": { - "default_category": "Network activity", - "to_ids": 1 - }, - "bro": { - "default_category": "Network activity", - "to_ids": 1 - }, - "pattern-in-file": { - "default_category": "Payload installation", - "to_ids": 1 - }, - "pattern-in-traffic": { - "default_category": "Network activity", - "to_ids": 1 - }, - "pattern-in-memory": { - "default_category": "Payload installation", - "to_ids": 1 - }, - "yara": { - "default_category": "Payload installation", - "to_ids": 1 - }, - "stix2-pattern": { - "default_category": "Payload installation", - "to_ids": 1 - }, - "sigma": { - "default_category": "Payload installation", - "to_ids": 1 - }, - "gene": { - "default_category": "Artifacts dropped", - "to_ids": 0 - }, - "mime-type": { - "default_category": "Artifacts dropped", - "to_ids": 0 - }, - "identity-card-number": { - "default_category": "Person", - "to_ids": 0 - }, - "cookie": { - "default_category": "Network activity", - "to_ids": 0 - }, - "vulnerability": { - "default_category": "External analysis", - "to_ids": 0 - }, - "attachment": { - "default_category": "External analysis", - "to_ids": 0 - }, - "malware-sample": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "link": { - "default_category": "External analysis", - "to_ids": 0 - }, - "comment": { - "default_category": "Other", - "to_ids": 0 - }, - "text": { - "default_category": "Other", - "to_ids": 0 - }, - "hex": { - "default_category": "Other", - "to_ids": 0 - }, - "other": { - "default_category": "Other", - "to_ids": 0 - }, - "named pipe": { - "default_category": "Artifacts dropped", - "to_ids": 0 - }, - "mutex": { - "default_category": "Artifacts dropped", - "to_ids": 1 - }, - "target-user": { - "default_category": "Targeting data", - "to_ids": 0 - }, - "target-email": { - "default_category": "Targeting data", - "to_ids": 0 - }, - "target-machine": { - "default_category": "Targeting data", - "to_ids": 0 - }, - "target-org": { - "default_category": "Targeting data", - "to_ids": 0 - }, - "target-location": { - "default_category": "Targeting data", - "to_ids": 0 - }, - "target-external": { - "default_category": "Targeting data", - "to_ids": 0 - }, - "btc": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "xmr": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "iban": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "bic": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "bank-account-nr": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "aba-rtn": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "bin": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "cc-number": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "prtn": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "phone-number": { - "default_category": "Person", - "to_ids": 0 - }, - "threat-actor": { - "default_category": "Attribution", - "to_ids": 0 - }, - "campaign-name": { - "default_category": "Attribution", - "to_ids": 0 - }, - "campaign-id": { - "default_category": "Attribution", - "to_ids": 0 - }, - "malware-type": { + "email-dst-display-name": { "default_category": "Payload delivery", "to_ids": 0 }, - "uri": { - "default_category": "Network activity", - "to_ids": 1 + "email-header": { + "default_category": "Payload delivery", + "to_ids": 0 }, - "authentihash": { + "email-message-id": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "email-mime-boundary": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "email-reply-to": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "email-src": { "default_category": "Payload delivery", "to_ids": 1 }, - "ssdeep": { + "email-src-display-name": { "default_category": "Payload delivery", - "to_ids": 1 + "to_ids": 0 }, - "imphash": { + "email-subject": { "default_category": "Payload delivery", - "to_ids": 1 + "to_ids": 0 }, - "pehash": { + "email-thread-index": { "default_category": "Payload delivery", - "to_ids": 1 + "to_ids": 0 }, - "impfuzzy": { + "email-x-mailer": { "default_category": "Payload delivery", - "to_ids": 1 + "to_ids": 0 }, - "sha224": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "sha384": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "sha512": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "sha512/224": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "sha512/256": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "tlsh": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "cdhash": { + "filename": { "default_category": "Payload delivery", "to_ids": 1 }, @@ -325,7 +577,7 @@ "default_category": "Payload delivery", "to_ids": 1 }, - "filename|ssdeep": { + "filename|impfuzzy": { "default_category": "Payload delivery", "to_ids": 1 }, @@ -333,7 +585,7 @@ "default_category": "Payload delivery", "to_ids": 1 }, - "filename|impfuzzy": { + "filename|md5": { "default_category": "Payload delivery", "to_ids": 1 }, @@ -341,10 +593,18 @@ "default_category": "Payload delivery", "to_ids": 1 }, + "filename|sha1": { + "default_category": "Payload delivery", + "to_ids": 1 + }, "filename|sha224": { "default_category": "Payload delivery", "to_ids": 1 }, + "filename|sha256": { + "default_category": "Payload delivery", + "to_ids": 1 + }, "filename|sha384": { "default_category": "Payload delivery", "to_ids": 1 @@ -361,94 +621,114 @@ "default_category": "Payload delivery", "to_ids": 1 }, + "filename|ssdeep": { + "default_category": "Payload delivery", + "to_ids": 1 + }, "filename|tlsh": { "default_category": "Payload delivery", "to_ids": 1 }, - "windows-scheduled-task": { + "first-name": { + "default_category": "Person", + "to_ids": 0 + }, + "float": { + "default_category": "Other", + "to_ids": 0 + }, + "frequent-flyer-number": { + "default_category": "Person", + "to_ids": 0 + }, + "gender": { + "default_category": "Person", + "to_ids": 0 + }, + "gene": { "default_category": "Artifacts dropped", "to_ids": 0 }, - "windows-service-name": { - "default_category": "Artifacts dropped", + "github-organisation": { + "default_category": "Social network", "to_ids": 0 }, - "windows-service-displayname": { - "default_category": "Artifacts dropped", + "github-repository": { + "default_category": "Social network", "to_ids": 0 }, - "whois-registrant-email": { - "default_category": "Attribution", + "github-username": { + "default_category": "Social network", "to_ids": 0 }, - "whois-registrant-phone": { - "default_category": "Attribution", + "hex": { + "default_category": "Other", "to_ids": 0 }, - "whois-registrant-name": { - "default_category": "Attribution", - "to_ids": 0 - }, - "whois-registrant-org": { - "default_category": "Attribution", - "to_ids": 0 - }, - "whois-registrar": { - "default_category": "Attribution", - "to_ids": 0 - }, - "whois-creation-date": { - "default_category": "Attribution", - "to_ids": 0 - }, - "x509-fingerprint-sha1": { + "hostname": { "default_category": "Network activity", "to_ids": 1 }, - "x509-fingerprint-md5": { + "hostname|port": { "default_category": "Network activity", "to_ids": 1 }, - "x509-fingerprint-sha256": { + "http-method": { "default_category": "Network activity", + "to_ids": 0 + }, + "iban": { + "default_category": "Financial fraud", "to_ids": 1 }, - "dns-soa-email": { - "default_category": "Attribution", + "identity-card-number": { + "default_category": "Person", "to_ids": 0 }, - "size-in-bytes": { - "default_category": "Other", - "to_ids": 0 + "impfuzzy": { + "default_category": "Payload delivery", + "to_ids": 1 }, - "counter": { - "default_category": "Other", - "to_ids": 0 + "imphash": { + "default_category": "Payload delivery", + "to_ids": 1 }, - "datetime": { - "default_category": "Other", - "to_ids": 0 - }, - "cpe": { - "default_category": "Other", - "to_ids": 0 - }, - "port": { + "ip-dst": { "default_category": "Network activity", - "to_ids": 0 + "to_ids": 1 }, "ip-dst|port": { "default_category": "Network activity", "to_ids": 1 }, + "ip-src": { + "default_category": "Network activity", + "to_ids": 1 + }, "ip-src|port": { "default_category": "Network activity", "to_ids": 1 }, - "hostname|port": { + "issue-date-of-the-visa": { + "default_category": "Person", + "to_ids": 0 + }, + "ja3-fingerprint-md5": { "default_category": "Network activity", "to_ids": 1 }, + "jabber-id": { + "default_category": "Social network", + "to_ids": 0 + }, + "last-name": { + "default_category": "Person", + "to_ids": 0 + }, + "link": { + "default_category": "External analysis", + "to_ids": 0 + }, "mac-address": { "default_category": "Network activity", "to_ids": 0 @@ -457,83 +737,47 @@ "default_category": "Network activity", "to_ids": 0 }, - "email-dst-display-name": { + "malware-sample": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "malware-type": { "default_category": "Payload delivery", "to_ids": 0 }, - "email-src-display-name": { + "md5": { "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-header": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-reply-to": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-x-mailer": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-mime-boundary": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-thread-index": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-message-id": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "github-username": { - "default_category": "Social network", - "to_ids": 0 - }, - "github-repository": { - "default_category": "Social network", - "to_ids": 0 - }, - "github-organisation": { - "default_category": "Social network", - "to_ids": 0 - }, - "jabber-id": { - "default_category": "Social network", - "to_ids": 0 - }, - "twitter-id": { - "default_category": "Social network", - "to_ids": 0 - }, - "first-name": { - "default_category": "Person", - "to_ids": 0 + "to_ids": 1 }, "middle-name": { "default_category": "Person", "to_ids": 0 }, - "last-name": { + "mime-type": { + "default_category": "Artifacts dropped", + "to_ids": 0 + }, + "mobile-application-id": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "mutex": { + "default_category": "Artifacts dropped", + "to_ids": 1 + }, + "named pipe": { + "default_category": "Artifacts dropped", + "to_ids": 0 + }, + "nationality": { "default_category": "Person", "to_ids": 0 }, - "date-of-birth": { - "default_category": "Person", + "other": { + "default_category": "Other", "to_ids": 0 }, - "place-of-birth": { - "default_category": "Person", - "to_ids": 0 - }, - "gender": { - "default_category": "Person", - "to_ids": 0 - }, - "passport-number": { + "passenger-name-record-locator-number": { "default_category": "Person", "to_ids": 0 }, @@ -545,47 +789,39 @@ "default_category": "Person", "to_ids": 0 }, - "redress-number": { + "passport-number": { "default_category": "Person", "to_ids": 0 }, - "nationality": { - "default_category": "Person", - "to_ids": 0 + "pattern-in-file": { + "default_category": "Payload installation", + "to_ids": 1 }, - "visa-number": { - "default_category": "Person", - "to_ids": 0 + "pattern-in-memory": { + "default_category": "Payload installation", + "to_ids": 1 }, - "issue-date-of-the-visa": { - "default_category": "Person", - "to_ids": 0 - }, - "primary-residence": { - "default_category": "Person", - "to_ids": 0 - }, - "country-of-residence": { - "default_category": "Person", - "to_ids": 0 - }, - "special-service-request": { - "default_category": "Person", - "to_ids": 0 - }, - "frequent-flyer-number": { - "default_category": "Person", - "to_ids": 0 - }, - "travel-details": { - "default_category": "Person", - "to_ids": 0 + "pattern-in-traffic": { + "default_category": "Network activity", + "to_ids": 1 }, "payment-details": { "default_category": "Person", "to_ids": 0 }, - "place-port-of-original-embarkation": { + "pdb": { + "default_category": "Artifacts dropped", + "to_ids": 0 + }, + "pehash": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "phone-number": { + "default_category": "Person", + "to_ids": 0 + }, + "place-of-birth": { "default_category": "Person", "to_ids": 0 }, @@ -597,590 +833,362 @@ "default_category": "Person", "to_ids": 0 }, - "passenger-name-record-locator-number": { + "place-port-of-original-embarkation": { "default_category": "Person", "to_ids": 0 }, - "mobile-application-id": { + "port": { + "default_category": "Network activity", + "to_ids": 0 + }, + "primary-residence": { + "default_category": "Person", + "to_ids": 0 + }, + "prtn": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "redress-number": { + "default_category": "Person", + "to_ids": 0 + }, + "regkey": { + "default_category": "Persistence mechanism", + "to_ids": 1 + }, + "regkey|value": { + "default_category": "Persistence mechanism", + "to_ids": 1 + }, + "sha1": { "default_category": "Payload delivery", "to_ids": 1 }, - "cortex": { + "sha224": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "sha256": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "sha384": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "sha512": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "sha512/224": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "sha512/256": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "sigma": { + "default_category": "Payload installation", + "to_ids": 1 + }, + "size-in-bytes": { + "default_category": "Other", + "to_ids": 0 + }, + "snort": { + "default_category": "Network activity", + "to_ids": 1 + }, + "special-service-request": { + "default_category": "Person", + "to_ids": 0 + }, + "ssdeep": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "stix2-pattern": { + "default_category": "Payload installation", + "to_ids": 1 + }, + "target-email": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "target-external": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "target-location": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "target-machine": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "target-org": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "target-user": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "text": { + "default_category": "Other", + "to_ids": 0 + }, + "threat-actor": { + "default_category": "Attribution", + "to_ids": 0 + }, + "tlsh": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "travel-details": { + "default_category": "Person", + "to_ids": 0 + }, + "twitter-id": { + "default_category": "Social network", + "to_ids": 0 + }, + "uri": { + "default_category": "Network activity", + "to_ids": 1 + }, + "url": { + "default_category": "Network activity", + "to_ids": 1 + }, + "user-agent": { + "default_category": "Network activity", + "to_ids": 0 + }, + "visa-number": { + "default_category": "Person", + "to_ids": 0 + }, + "vulnerability": { "default_category": "External analysis", "to_ids": 0 }, - "boolean": { - "default_category": "Other", + "whois-creation-date": { + "default_category": "Attribution", "to_ids": 0 + }, + "whois-registrant-email": { + "default_category": "Attribution", + "to_ids": 0 + }, + "whois-registrant-name": { + "default_category": "Attribution", + "to_ids": 0 + }, + "whois-registrant-org": { + "default_category": "Attribution", + "to_ids": 0 + }, + "whois-registrant-phone": { + "default_category": "Attribution", + "to_ids": 0 + }, + "whois-registrar": { + "default_category": "Attribution", + "to_ids": 0 + }, + "windows-scheduled-task": { + "default_category": "Artifacts dropped", + "to_ids": 0 + }, + "windows-service-displayname": { + "default_category": "Artifacts dropped", + "to_ids": 0 + }, + "windows-service-name": { + "default_category": "Artifacts dropped", + "to_ids": 0 + }, + "x509-fingerprint-md5": { + "default_category": "Network activity", + "to_ids": 1 + }, + "x509-fingerprint-sha1": { + "default_category": "Network activity", + "to_ids": 1 + }, + "x509-fingerprint-sha256": { + "default_category": "Network activity", + "to_ids": 1 + }, + "xmr": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "yara": { + "default_category": "Payload installation", + "to_ids": 1 } }, "types": [ - "md5", - "sha1", - "sha256", - "filename", - "pdb", - "filename|md5", - "filename|sha1", - "filename|sha256", - "ip-src", - "ip-dst", - "hostname", + "AS", + "aba-rtn", + "attachment", + "authentihash", + "bank-account-nr", + "bic", + "bin", + "boolean", + "bro", + "btc", + "campaign-id", + "campaign-name", + "cc-number", + "cdhash", + "comment", + "cookie", + "cortex", + "counter", + "country-of-residence", + "cpe", + "date-of-birth", + "datetime", + "dns-soa-email", "domain", "domain|ip", - "email-src", - "email-dst", - "email-subject", "email-attachment", "email-body", - "float", - "url", - "http-method", - "user-agent", - "regkey", - "regkey|value", - "AS", - "snort", - "bro", - "pattern-in-file", - "pattern-in-traffic", - "pattern-in-memory", - "yara", - "stix2-pattern", - "sigma", - "gene", - "mime-type", - "identity-card-number", - "cookie", - "vulnerability", - "attachment", - "malware-sample", - "link", - "comment", - "text", - "hex", - "other", - "named pipe", - "mutex", - "target-user", - "target-email", - "target-machine", - "target-org", - "target-location", - "target-external", - "btc", - "xmr", - "iban", - "bic", - "bank-account-nr", - "aba-rtn", - "bin", - "cc-number", - "prtn", - "phone-number", - "threat-actor", - "campaign-name", - "campaign-id", - "malware-type", - "uri", - "authentihash", - "ssdeep", - "imphash", - "pehash", - "impfuzzy", - "sha224", - "sha384", - "sha512", - "sha512/224", - "sha512/256", - "tlsh", - "cdhash", + "email-dst", + "email-dst-display-name", + "email-header", + "email-message-id", + "email-mime-boundary", + "email-reply-to", + "email-src", + "email-src-display-name", + "email-subject", + "email-thread-index", + "email-x-mailer", + "filename", "filename|authentihash", - "filename|ssdeep", - "filename|imphash", "filename|impfuzzy", + "filename|imphash", + "filename|md5", "filename|pehash", + "filename|sha1", "filename|sha224", + "filename|sha256", "filename|sha384", "filename|sha512", "filename|sha512/224", "filename|sha512/256", + "filename|ssdeep", "filename|tlsh", - "windows-scheduled-task", - "windows-service-name", - "windows-service-displayname", - "whois-registrant-email", - "whois-registrant-phone", - "whois-registrant-name", - "whois-registrant-org", - "whois-registrar", - "whois-creation-date", - "x509-fingerprint-sha1", - "x509-fingerprint-md5", - "x509-fingerprint-sha256", - "dns-soa-email", - "size-in-bytes", - "counter", - "datetime", - "cpe", - "port", - "ip-dst|port", - "ip-src|port", + "first-name", + "float", + "frequent-flyer-number", + "gender", + "gene", + "github-organisation", + "github-repository", + "github-username", + "hex", + "hostname", "hostname|port", + "http-method", + "iban", + "identity-card-number", + "impfuzzy", + "imphash", + "ip-dst", + "ip-dst|port", + "ip-src", + "ip-src|port", + "issue-date-of-the-visa", + "ja3-fingerprint-md5", + "jabber-id", + "last-name", + "link", "mac-address", "mac-eui-64", - "email-dst-display-name", - "email-src-display-name", - "email-header", - "email-reply-to", - "email-x-mailer", - "email-mime-boundary", - "email-thread-index", - "email-message-id", - "github-username", - "github-repository", - "github-organisation", - "jabber-id", - "twitter-id", - "first-name", + "malware-sample", + "malware-type", + "md5", "middle-name", - "last-name", - "date-of-birth", - "place-of-birth", - "gender", - "passport-number", + "mime-type", + "mobile-application-id", + "mutex", + "named pipe", + "nationality", + "other", + "passenger-name-record-locator-number", "passport-country", "passport-expiration", - "redress-number", - "nationality", - "visa-number", - "issue-date-of-the-visa", - "primary-residence", - "country-of-residence", - "special-service-request", - "frequent-flyer-number", - "travel-details", + "passport-number", + "pattern-in-file", + "pattern-in-memory", + "pattern-in-traffic", "payment-details", - "place-port-of-original-embarkation", + "pdb", + "pehash", + "phone-number", + "place-of-birth", "place-port-of-clearance", "place-port-of-onward-foreign-destination", - "passenger-name-record-locator-number", - "mobile-application-id", - "cortex", - "boolean" - ], - "categories": [ - "Internal reference", - "Targeting data", - "Antivirus detection", - "Payload delivery", - "Artifacts dropped", - "Payload installation", - "Persistence mechanism", - "Network activity", - "Payload type", - "Attribution", - "External analysis", - "Financial fraud", - "Support Tool", - "Social network", - "Person", - "Other" - ], - "category_type_mappings": { - "Internal reference": [ - "text", - "link", - "comment", - "other", - "hex" - ], - "Targeting data": [ - "target-user", - "target-email", - "target-machine", - "target-org", - "target-location", - "target-external", - "comment" - ], - "Antivirus detection": [ - "link", - "comment", - "text", - "hex", - "attachment", - "other" - ], - "Payload delivery": [ - "md5", - "sha1", - "sha224", - "sha256", - "sha384", - "sha512", - "sha512/224", - "sha512/256", - "ssdeep", - "imphash", - "impfuzzy", - "authentihash", - "pehash", - "tlsh", - "cdhash", - "filename", - "filename|md5", - "filename|sha1", - "filename|sha224", - "filename|sha256", - "filename|sha384", - "filename|sha512", - "filename|sha512/224", - "filename|sha512/256", - "filename|authentihash", - "filename|ssdeep", - "filename|tlsh", - "filename|imphash", - "filename|impfuzzy", - "filename|pehash", - "mac-address", - "mac-eui-64", - "ip-src", - "ip-dst", - "ip-dst|port", - "ip-src|port", - "hostname", - "domain", - "email-src", - "email-dst", - "email-subject", - "email-attachment", - "email-body", - "url", - "user-agent", - "AS", - "pattern-in-file", - "pattern-in-traffic", - "stix2-pattern", - "yara", - "sigma", - "mime-type", - "attachment", - "malware-sample", - "link", - "malware-type", - "comment", - "text", - "hex", - "vulnerability", - "x509-fingerprint-sha1", - "x509-fingerprint-md5", - "x509-fingerprint-sha256", - "other", - "hostname|port", - "email-dst-display-name", - "email-src-display-name", - "email-header", - "email-reply-to", - "email-x-mailer", - "email-mime-boundary", - "email-thread-index", - "email-message-id", - "mobile-application-id", - "whois-registrant-email" - ], - "Artifacts dropped": [ - "md5", - "sha1", - "sha224", - "sha256", - "sha384", - "sha512", - "sha512/224", - "sha512/256", - "ssdeep", - "imphash", - "impfuzzy", - "authentihash", - "cdhash", - "filename", - "filename|md5", - "filename|sha1", - "filename|sha224", - "filename|sha256", - "filename|sha384", - "filename|sha512", - "filename|sha512/224", - "filename|sha512/256", - "filename|authentihash", - "filename|ssdeep", - "filename|tlsh", - "filename|imphash", - "filename|impfuzzy", - "filename|pehash", - "regkey", - "regkey|value", - "pattern-in-file", - "pattern-in-memory", - "pdb", - "stix2-pattern", - "yara", - "sigma", - "attachment", - "malware-sample", - "named pipe", - "mutex", - "windows-scheduled-task", - "windows-service-name", - "windows-service-displayname", - "comment", - "text", - "hex", - "x509-fingerprint-sha1", - "x509-fingerprint-md5", - "x509-fingerprint-sha256", - "other", - "cookie", - "gene", - "mime-type" - ], - "Payload installation": [ - "md5", - "sha1", - "sha224", - "sha256", - "sha384", - "sha512", - "sha512/224", - "sha512/256", - "ssdeep", - "imphash", - "impfuzzy", - "authentihash", - "pehash", - "tlsh", - "cdhash", - "filename", - "filename|md5", - "filename|sha1", - "filename|sha224", - "filename|sha256", - "filename|sha384", - "filename|sha512", - "filename|sha512/224", - "filename|sha512/256", - "filename|authentihash", - "filename|ssdeep", - "filename|tlsh", - "filename|imphash", - "filename|impfuzzy", - "filename|pehash", - "pattern-in-file", - "pattern-in-traffic", - "pattern-in-memory", - "stix2-pattern", - "yara", - "sigma", - "vulnerability", - "attachment", - "malware-sample", - "malware-type", - "comment", - "text", - "hex", - "x509-fingerprint-sha1", - "x509-fingerprint-md5", - "x509-fingerprint-sha256", - "mobile-application-id", - "other", - "mime-type" - ], - "Persistence mechanism": [ - "filename", - "regkey", - "regkey|value", - "comment", - "text", - "other", - "hex" - ], - "Network activity": [ - "ip-src", - "ip-dst", - "ip-dst|port", - "ip-src|port", - "port", - "hostname", - "domain", - "domain|ip", - "mac-address", - "mac-eui-64", - "email-dst", - "url", - "uri", - "user-agent", - "http-method", - "AS", - "snort", - "pattern-in-file", - "stix2-pattern", - "pattern-in-traffic", - "attachment", - "comment", - "text", - "x509-fingerprint-md5", - "x509-fingerprint-sha1", - "x509-fingerprint-sha256", - "other", - "hex", - "cookie", - "hostname|port", - "bro" - ], - "Payload type": [ - "comment", - "text", - "other" - ], - "Attribution": [ - "threat-actor", - "campaign-name", - "campaign-id", - "whois-registrant-phone", - "whois-registrant-email", - "whois-registrant-name", - "whois-registrant-org", - "whois-registrar", - "whois-creation-date", - "comment", - "text", - "x509-fingerprint-sha1", - "x509-fingerprint-md5", - "x509-fingerprint-sha256", - "other", - "dns-soa-email" - ], - "External analysis": [ - "md5", - "sha1", - "sha256", - "filename", - "filename|md5", - "filename|sha1", - "filename|sha256", - "ip-src", - "ip-dst", - "ip-dst|port", - "ip-src|port", - "mac-address", - "mac-eui-64", - "hostname", - "domain", - "domain|ip", - "url", - "user-agent", - "regkey", - "regkey|value", - "AS", - "snort", - "bro", - "pattern-in-file", - "pattern-in-traffic", - "pattern-in-memory", - "vulnerability", - "attachment", - "malware-sample", - "link", - "comment", - "text", - "x509-fingerprint-sha1", - "x509-fingerprint-md5", - "x509-fingerprint-sha256", - "github-repository", - "other", - "cortex" - ], - "Financial fraud": [ - "btc", - "xmr", - "iban", - "bic", - "bank-account-nr", - "aba-rtn", - "bin", - "cc-number", - "prtn", - "phone-number", - "comment", - "text", - "other", - "hex" - ], - "Support Tool": [ - "link", - "text", - "attachment", - "comment", - "other", - "hex" - ], - "Social network": [ - "github-username", - "github-repository", - "github-organisation", - "jabber-id", - "twitter-id", - "email-src", - "email-dst", - "comment", - "text", - "other", - "whois-registrant-email" - ], - "Person": [ - "first-name", - "middle-name", - "last-name", - "date-of-birth", - "place-of-birth", - "gender", - "passport-number", - "passport-country", - "passport-expiration", - "redress-number", - "nationality", - "visa-number", - "issue-date-of-the-visa", - "primary-residence", - "country-of-residence", - "special-service-request", - "frequent-flyer-number", - "travel-details", - "payment-details", - "place-port-of-original-embarkation", - "place-port-of-clearance", - "place-port-of-onward-foreign-destination", - "passenger-name-record-locator-number", - "comment", - "text", - "other", - "phone-number", - "identity-card-number" - ], - "Other": [ - "comment", - "text", - "other", - "size-in-bytes", - "counter", - "datetime", - "cpe", - "port", - "float", - "hex", - "phone-number", - "boolean" - ] - } + "place-port-of-original-embarkation", + "port", + "primary-residence", + "prtn", + "redress-number", + "regkey", + "regkey|value", + "sha1", + "sha224", + "sha256", + "sha384", + "sha512", + "sha512/224", + "sha512/256", + "sigma", + "size-in-bytes", + "snort", + "special-service-request", + "ssdeep", + "stix2-pattern", + "target-email", + "target-external", + "target-location", + "target-machine", + "target-org", + "target-user", + "text", + "threat-actor", + "tlsh", + "travel-details", + "twitter-id", + "uri", + "url", + "user-agent", + "visa-number", + "vulnerability", + "whois-creation-date", + "whois-registrant-email", + "whois-registrant-name", + "whois-registrant-org", + "whois-registrant-phone", + "whois-registrar", + "windows-scheduled-task", + "windows-service-displayname", + "windows-service-name", + "x509-fingerprint-md5", + "x509-fingerprint-sha1", + "x509-fingerprint-sha256", + "xmr", + "yara" + ] } } From e3bc4f2be65f395f94008c34979c2df3f946241e Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Sun, 30 Dec 2018 13:02:14 +0100 Subject: [PATCH 32/38] chg: [data] describeTypes updated (grabbed from MISP HEAD) --- pymisp/data/describeTypes.json | 2086 ++++++++++++++++---------------- 1 file changed, 1043 insertions(+), 1043 deletions(-) diff --git a/pymisp/data/describeTypes.json b/pymisp/data/describeTypes.json index 850668d..5f2bec3 100644 --- a/pymisp/data/describeTypes.json +++ b/pymisp/data/describeTypes.json @@ -1,513 +1,49 @@ { "result": { - "categories": [ - "Antivirus detection", - "Artifacts dropped", - "Attribution", - "External analysis", - "Financial fraud", - "Internal reference", - "Network activity", - "Other", - "Payload delivery", - "Payload installation", - "Payload type", - "Persistence mechanism", - "Person", - "Social network", - "Support Tool", - "Targeting data" - ], - "category_type_mappings": { - "Antivirus detection": [ - "attachment", - "comment", - "hex", - "link", - "other", - "text" - ], - "Artifacts dropped": [ - "attachment", - "authentihash", - "cdhash", - "comment", - "cookie", - "filename", - "filename|authentihash", - "filename|impfuzzy", - "filename|imphash", - "filename|md5", - "filename|pehash", - "filename|sha1", - "filename|sha224", - "filename|sha256", - "filename|sha384", - "filename|sha512", - "filename|sha512/224", - "filename|sha512/256", - "filename|ssdeep", - "filename|tlsh", - "gene", - "hex", - "impfuzzy", - "imphash", - "malware-sample", - "md5", - "mime-type", - "mutex", - "named pipe", - "other", - "pattern-in-file", - "pattern-in-memory", - "pdb", - "regkey", - "regkey|value", - "sha1", - "sha224", - "sha256", - "sha384", - "sha512", - "sha512/224", - "sha512/256", - "sigma", - "ssdeep", - "stix2-pattern", - "text", - "windows-scheduled-task", - "windows-service-displayname", - "windows-service-name", - "x509-fingerprint-md5", - "x509-fingerprint-sha1", - "x509-fingerprint-sha256", - "yara" - ], - "Attribution": [ - "campaign-id", - "campaign-name", - "comment", - "dns-soa-email", - "other", - "text", - "threat-actor", - "whois-creation-date", - "whois-registrant-email", - "whois-registrant-name", - "whois-registrant-org", - "whois-registrant-phone", - "whois-registrar", - "x509-fingerprint-md5", - "x509-fingerprint-sha1", - "x509-fingerprint-sha256" - ], - "External analysis": [ - "AS", - "attachment", - "bro", - "comment", - "cortex", - "domain", - "domain|ip", - "filename", - "filename|md5", - "filename|sha1", - "filename|sha256", - "github-repository", - "hostname", - "ip-dst", - "ip-dst|port", - "ip-src", - "ip-src|port", - "ja3-fingerprint-md5", - "link", - "mac-address", - "mac-eui-64", - "malware-sample", - "md5", - "other", - "pattern-in-file", - "pattern-in-memory", - "pattern-in-traffic", - "regkey", - "regkey|value", - "sha1", - "sha256", - "snort", - "text", - "url", - "user-agent", - "vulnerability", - "x509-fingerprint-md5", - "x509-fingerprint-sha1", - "x509-fingerprint-sha256" - ], - "Financial fraud": [ - "aba-rtn", - "bank-account-nr", - "bic", - "bin", - "btc", - "cc-number", - "comment", - "hex", - "iban", - "other", - "phone-number", - "prtn", - "text", - "xmr" - ], - "Internal reference": [ - "comment", - "hex", - "link", - "other", - "text" - ], - "Network activity": [ - "AS", - "attachment", - "bro", - "comment", - "cookie", - "domain", - "domain|ip", - "email-dst", - "hex", - "hostname", - "hostname|port", - "http-method", - "ip-dst", - "ip-dst|port", - "ip-src", - "ip-src|port", - "ja3-fingerprint-md5", - "mac-address", - "mac-eui-64", - "other", - "pattern-in-file", - "pattern-in-traffic", - "port", - "snort", - "stix2-pattern", - "text", - "uri", - "url", - "user-agent", - "x509-fingerprint-md5", - "x509-fingerprint-sha1", - "x509-fingerprint-sha256" - ], - "Other": [ - "boolean", - "comment", - "counter", - "cpe", - "datetime", - "float", - "hex", - "other", - "phone-number", - "port", - "size-in-bytes", - "text" - ], - "Payload delivery": [ - "AS", - "attachment", - "authentihash", - "cdhash", - "comment", - "domain", - "email-attachment", - "email-body", - "email-dst", - "email-dst-display-name", - "email-header", - "email-message-id", - "email-mime-boundary", - "email-reply-to", - "email-src", - "email-src-display-name", - "email-subject", - "email-thread-index", - "email-x-mailer", - "filename", - "filename|authentihash", - "filename|impfuzzy", - "filename|imphash", - "filename|md5", - "filename|pehash", - "filename|sha1", - "filename|sha224", - "filename|sha256", - "filename|sha384", - "filename|sha512", - "filename|sha512/224", - "filename|sha512/256", - "filename|ssdeep", - "filename|tlsh", - "hex", - "hostname", - "hostname|port", - "impfuzzy", - "imphash", - "ip-dst", - "ip-dst|port", - "ip-src", - "ip-src|port", - "ja3-fingerprint-md5", - "link", - "mac-address", - "mac-eui-64", - "malware-sample", - "malware-type", - "md5", - "mime-type", - "mobile-application-id", - "other", - "pattern-in-file", - "pattern-in-traffic", - "pehash", - "sha1", - "sha224", - "sha256", - "sha384", - "sha512", - "sha512/224", - "sha512/256", - "sigma", - "ssdeep", - "stix2-pattern", - "text", - "tlsh", - "url", - "user-agent", - "vulnerability", - "whois-registrant-email", - "x509-fingerprint-md5", - "x509-fingerprint-sha1", - "x509-fingerprint-sha256", - "yara" - ], - "Payload installation": [ - "attachment", - "authentihash", - "cdhash", - "comment", - "filename", - "filename|authentihash", - "filename|impfuzzy", - "filename|imphash", - "filename|md5", - "filename|pehash", - "filename|sha1", - "filename|sha224", - "filename|sha256", - "filename|sha384", - "filename|sha512", - "filename|sha512/224", - "filename|sha512/256", - "filename|ssdeep", - "filename|tlsh", - "hex", - "impfuzzy", - "imphash", - "malware-sample", - "malware-type", - "md5", - "mime-type", - "mobile-application-id", - "other", - "pattern-in-file", - "pattern-in-memory", - "pattern-in-traffic", - "pehash", - "sha1", - "sha224", - "sha256", - "sha384", - "sha512", - "sha512/224", - "sha512/256", - "sigma", - "ssdeep", - "stix2-pattern", - "text", - "tlsh", - "vulnerability", - "x509-fingerprint-md5", - "x509-fingerprint-sha1", - "x509-fingerprint-sha256", - "yara" - ], - "Payload type": [ - "comment", - "other", - "text" - ], - "Persistence mechanism": [ - "comment", - "filename", - "hex", - "other", - "regkey", - "regkey|value", - "text" - ], - "Person": [ - "comment", - "country-of-residence", - "date-of-birth", - "first-name", - "frequent-flyer-number", - "gender", - "identity-card-number", - "issue-date-of-the-visa", - "last-name", - "middle-name", - "nationality", - "other", - "passenger-name-record-locator-number", - "passport-country", - "passport-expiration", - "passport-number", - "payment-details", - "phone-number", - "place-of-birth", - "place-port-of-clearance", - "place-port-of-onward-foreign-destination", - "place-port-of-original-embarkation", - "primary-residence", - "redress-number", - "special-service-request", - "text", - "travel-details", - "visa-number" - ], - "Social network": [ - "comment", - "email-dst", - "email-src", - "github-organisation", - "github-repository", - "github-username", - "jabber-id", - "other", - "text", - "twitter-id", - "whois-registrant-email" - ], - "Support Tool": [ - "attachment", - "comment", - "hex", - "link", - "other", - "text" - ], - "Targeting data": [ - "comment", - "target-email", - "target-external", - "target-location", - "target-machine", - "target-org", - "target-user" - ] - }, "sane_defaults": { - "AS": { - "default_category": "Network activity", - "to_ids": 0 - }, - "aba-rtn": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "attachment": { - "default_category": "External analysis", - "to_ids": 0 - }, - "authentihash": { + "md5": { "default_category": "Payload delivery", "to_ids": 1 }, - "bank-account-nr": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "bic": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "bin": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "boolean": { - "default_category": "Other", - "to_ids": 0 - }, - "bro": { - "default_category": "Network activity", - "to_ids": 1 - }, - "btc": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "campaign-id": { - "default_category": "Attribution", - "to_ids": 0 - }, - "campaign-name": { - "default_category": "Attribution", - "to_ids": 0 - }, - "cc-number": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "cdhash": { + "sha1": { "default_category": "Payload delivery", "to_ids": 1 }, - "comment": { - "default_category": "Other", + "sha256": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "pdb": { + "default_category": "Artifacts dropped", "to_ids": 0 }, - "cookie": { + "filename|md5": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha1": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha256": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "ip-src": { "default_category": "Network activity", - "to_ids": 0 + "to_ids": 1 }, - "cortex": { - "default_category": "External analysis", - "to_ids": 0 + "ip-dst": { + "default_category": "Network activity", + "to_ids": 1 }, - "counter": { - "default_category": "Other", - "to_ids": 0 - }, - "country-of-residence": { - "default_category": "Person", - "to_ids": 0 - }, - "cpe": { - "default_category": "Other", - "to_ids": 0 - }, - "date-of-birth": { - "default_category": "Person", - "to_ids": 0 - }, - "datetime": { - "default_category": "Other", - "to_ids": 0 - }, - "dns-soa-email": { - "default_category": "Attribution", - "to_ids": 0 + "hostname": { + "default_category": "Network activity", + "to_ids": 1 }, "domain": { "default_category": "Network activity", @@ -517,6 +53,18 @@ "default_category": "Network activity", "to_ids": 1 }, + "email-src": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "email-dst": { + "default_category": "Network activity", + "to_ids": 1 + }, + "email-subject": { + "default_category": "Payload delivery", + "to_ids": 0 + }, "email-attachment": { "default_category": "Payload delivery", "to_ids": 1 @@ -525,151 +73,11 @@ "default_category": "Payload delivery", "to_ids": 0 }, - "email-dst": { - "default_category": "Network activity", - "to_ids": 1 - }, - "email-dst-display-name": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-header": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-message-id": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-mime-boundary": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-reply-to": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-src": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "email-src-display-name": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-subject": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-thread-index": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-x-mailer": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "filename": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|authentihash": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|impfuzzy": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|imphash": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|md5": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|pehash": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|sha1": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|sha224": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|sha256": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|sha384": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|sha512": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|sha512/224": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|sha512/256": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|ssdeep": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|tlsh": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "first-name": { - "default_category": "Person", - "to_ids": 0 - }, "float": { "default_category": "Other", "to_ids": 0 }, - "frequent-flyer-number": { - "default_category": "Person", - "to_ids": 0 - }, - "gender": { - "default_category": "Person", - "to_ids": 0 - }, - "gene": { - "default_category": "Artifacts dropped", - "to_ids": 0 - }, - "github-organisation": { - "default_category": "Social network", - "to_ids": 0 - }, - "github-repository": { - "default_category": "Social network", - "to_ids": 0 - }, - "github-username": { - "default_category": "Social network", - "to_ids": 0 - }, - "hex": { - "default_category": "Other", - "to_ids": 0 - }, - "hostname": { - "default_category": "Network activity", - "to_ids": 1 - }, - "hostname|port": { + "url": { "default_category": "Network activity", "to_ids": 1 }, @@ -677,182 +85,14 @@ "default_category": "Network activity", "to_ids": 0 }, - "iban": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "identity-card-number": { - "default_category": "Person", - "to_ids": 0 - }, - "impfuzzy": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "imphash": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "ip-dst": { + "user-agent": { "default_category": "Network activity", - "to_ids": 1 - }, - "ip-dst|port": { - "default_category": "Network activity", - "to_ids": 1 - }, - "ip-src": { - "default_category": "Network activity", - "to_ids": 1 - }, - "ip-src|port": { - "default_category": "Network activity", - "to_ids": 1 - }, - "issue-date-of-the-visa": { - "default_category": "Person", "to_ids": 0 }, "ja3-fingerprint-md5": { "default_category": "Network activity", "to_ids": 1 }, - "jabber-id": { - "default_category": "Social network", - "to_ids": 0 - }, - "last-name": { - "default_category": "Person", - "to_ids": 0 - }, - "link": { - "default_category": "External analysis", - "to_ids": 0 - }, - "mac-address": { - "default_category": "Network activity", - "to_ids": 0 - }, - "mac-eui-64": { - "default_category": "Network activity", - "to_ids": 0 - }, - "malware-sample": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "malware-type": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "md5": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "middle-name": { - "default_category": "Person", - "to_ids": 0 - }, - "mime-type": { - "default_category": "Artifacts dropped", - "to_ids": 0 - }, - "mobile-application-id": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "mutex": { - "default_category": "Artifacts dropped", - "to_ids": 1 - }, - "named pipe": { - "default_category": "Artifacts dropped", - "to_ids": 0 - }, - "nationality": { - "default_category": "Person", - "to_ids": 0 - }, - "other": { - "default_category": "Other", - "to_ids": 0 - }, - "passenger-name-record-locator-number": { - "default_category": "Person", - "to_ids": 0 - }, - "passport-country": { - "default_category": "Person", - "to_ids": 0 - }, - "passport-expiration": { - "default_category": "Person", - "to_ids": 0 - }, - "passport-number": { - "default_category": "Person", - "to_ids": 0 - }, - "pattern-in-file": { - "default_category": "Payload installation", - "to_ids": 1 - }, - "pattern-in-memory": { - "default_category": "Payload installation", - "to_ids": 1 - }, - "pattern-in-traffic": { - "default_category": "Network activity", - "to_ids": 1 - }, - "payment-details": { - "default_category": "Person", - "to_ids": 0 - }, - "pdb": { - "default_category": "Artifacts dropped", - "to_ids": 0 - }, - "pehash": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "phone-number": { - "default_category": "Person", - "to_ids": 0 - }, - "place-of-birth": { - "default_category": "Person", - "to_ids": 0 - }, - "place-port-of-clearance": { - "default_category": "Person", - "to_ids": 0 - }, - "place-port-of-onward-foreign-destination": { - "default_category": "Person", - "to_ids": 0 - }, - "place-port-of-original-embarkation": { - "default_category": "Person", - "to_ids": 0 - }, - "port": { - "default_category": "Network activity", - "to_ids": 0 - }, - "primary-residence": { - "default_category": "Person", - "to_ids": 0 - }, - "prtn": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "redress-number": { - "default_category": "Person", - "to_ids": 0 - }, "regkey": { "default_category": "Persistence mechanism", "to_ids": 1 @@ -861,7 +101,199 @@ "default_category": "Persistence mechanism", "to_ids": 1 }, - "sha1": { + "AS": { + "default_category": "Network activity", + "to_ids": 0 + }, + "snort": { + "default_category": "Network activity", + "to_ids": 1 + }, + "bro": { + "default_category": "Network activity", + "to_ids": 1 + }, + "pattern-in-file": { + "default_category": "Payload installation", + "to_ids": 1 + }, + "pattern-in-traffic": { + "default_category": "Network activity", + "to_ids": 1 + }, + "pattern-in-memory": { + "default_category": "Payload installation", + "to_ids": 1 + }, + "yara": { + "default_category": "Payload installation", + "to_ids": 1 + }, + "stix2-pattern": { + "default_category": "Payload installation", + "to_ids": 1 + }, + "sigma": { + "default_category": "Payload installation", + "to_ids": 1 + }, + "gene": { + "default_category": "Artifacts dropped", + "to_ids": 0 + }, + "mime-type": { + "default_category": "Artifacts dropped", + "to_ids": 0 + }, + "identity-card-number": { + "default_category": "Person", + "to_ids": 0 + }, + "cookie": { + "default_category": "Network activity", + "to_ids": 0 + }, + "vulnerability": { + "default_category": "External analysis", + "to_ids": 0 + }, + "attachment": { + "default_category": "External analysis", + "to_ids": 0 + }, + "malware-sample": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "link": { + "default_category": "External analysis", + "to_ids": 0 + }, + "comment": { + "default_category": "Other", + "to_ids": 0 + }, + "text": { + "default_category": "Other", + "to_ids": 0 + }, + "hex": { + "default_category": "Other", + "to_ids": 0 + }, + "other": { + "default_category": "Other", + "to_ids": 0 + }, + "named pipe": { + "default_category": "Artifacts dropped", + "to_ids": 0 + }, + "mutex": { + "default_category": "Artifacts dropped", + "to_ids": 1 + }, + "target-user": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "target-email": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "target-machine": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "target-org": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "target-location": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "target-external": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "btc": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "xmr": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "iban": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "bic": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "bank-account-nr": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "aba-rtn": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "bin": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "cc-number": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "prtn": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "phone-number": { + "default_category": "Person", + "to_ids": 0 + }, + "threat-actor": { + "default_category": "Attribution", + "to_ids": 0 + }, + "campaign-name": { + "default_category": "Attribution", + "to_ids": 0 + }, + "campaign-id": { + "default_category": "Attribution", + "to_ids": 0 + }, + "malware-type": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "uri": { + "default_category": "Network activity", + "to_ids": 1 + }, + "authentihash": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "ssdeep": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "imphash": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "pehash": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "impfuzzy": { "default_category": "Payload delivery", "to_ids": 1 }, @@ -869,10 +301,6 @@ "default_category": "Payload delivery", "to_ids": 1 }, - "sha256": { - "default_category": "Payload delivery", - "to_ids": 1 - }, "sha384": { "default_category": "Payload delivery", "to_ids": 1 @@ -889,102 +317,78 @@ "default_category": "Payload delivery", "to_ids": 1 }, - "sigma": { - "default_category": "Payload installation", - "to_ids": 1 - }, - "size-in-bytes": { - "default_category": "Other", - "to_ids": 0 - }, - "snort": { - "default_category": "Network activity", - "to_ids": 1 - }, - "special-service-request": { - "default_category": "Person", - "to_ids": 0 - }, - "ssdeep": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "stix2-pattern": { - "default_category": "Payload installation", - "to_ids": 1 - }, - "target-email": { - "default_category": "Targeting data", - "to_ids": 0 - }, - "target-external": { - "default_category": "Targeting data", - "to_ids": 0 - }, - "target-location": { - "default_category": "Targeting data", - "to_ids": 0 - }, - "target-machine": { - "default_category": "Targeting data", - "to_ids": 0 - }, - "target-org": { - "default_category": "Targeting data", - "to_ids": 0 - }, - "target-user": { - "default_category": "Targeting data", - "to_ids": 0 - }, - "text": { - "default_category": "Other", - "to_ids": 0 - }, - "threat-actor": { - "default_category": "Attribution", - "to_ids": 0 - }, "tlsh": { "default_category": "Payload delivery", "to_ids": 1 }, - "travel-details": { - "default_category": "Person", - "to_ids": 0 - }, - "twitter-id": { - "default_category": "Social network", - "to_ids": 0 - }, - "uri": { - "default_category": "Network activity", + "cdhash": { + "default_category": "Payload delivery", "to_ids": 1 }, - "url": { - "default_category": "Network activity", + "filename|authentihash": { + "default_category": "Payload delivery", "to_ids": 1 }, - "user-agent": { - "default_category": "Network activity", + "filename|ssdeep": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|imphash": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|impfuzzy": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|pehash": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha224": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha384": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha512": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha512/224": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha512/256": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|tlsh": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "windows-scheduled-task": { + "default_category": "Artifacts dropped", "to_ids": 0 }, - "visa-number": { - "default_category": "Person", + "windows-service-name": { + "default_category": "Artifacts dropped", "to_ids": 0 }, - "vulnerability": { - "default_category": "External analysis", - "to_ids": 0 - }, - "whois-creation-date": { - "default_category": "Attribution", + "windows-service-displayname": { + "default_category": "Artifacts dropped", "to_ids": 0 }, "whois-registrant-email": { "default_category": "Attribution", "to_ids": 0 }, + "whois-registrant-phone": { + "default_category": "Attribution", + "to_ids": 0 + }, "whois-registrant-name": { "default_category": "Attribution", "to_ids": 0 @@ -993,31 +397,19 @@ "default_category": "Attribution", "to_ids": 0 }, - "whois-registrant-phone": { - "default_category": "Attribution", - "to_ids": 0 - }, "whois-registrar": { "default_category": "Attribution", "to_ids": 0 }, - "windows-scheduled-task": { - "default_category": "Artifacts dropped", + "whois-creation-date": { + "default_category": "Attribution", "to_ids": 0 }, - "windows-service-displayname": { - "default_category": "Artifacts dropped", - "to_ids": 0 - }, - "windows-service-name": { - "default_category": "Artifacts dropped", - "to_ids": 0 - }, - "x509-fingerprint-md5": { + "x509-fingerprint-sha1": { "default_category": "Network activity", "to_ids": 1 }, - "x509-fingerprint-sha1": { + "x509-fingerprint-md5": { "default_category": "Network activity", "to_ids": 1 }, @@ -1025,170 +417,778 @@ "default_category": "Network activity", "to_ids": 1 }, - "xmr": { - "default_category": "Financial fraud", + "dns-soa-email": { + "default_category": "Attribution", + "to_ids": 0 + }, + "size-in-bytes": { + "default_category": "Other", + "to_ids": 0 + }, + "counter": { + "default_category": "Other", + "to_ids": 0 + }, + "datetime": { + "default_category": "Other", + "to_ids": 0 + }, + "cpe": { + "default_category": "Other", + "to_ids": 0 + }, + "port": { + "default_category": "Network activity", + "to_ids": 0 + }, + "ip-dst|port": { + "default_category": "Network activity", "to_ids": 1 }, - "yara": { - "default_category": "Payload installation", + "ip-src|port": { + "default_category": "Network activity", "to_ids": 1 + }, + "hostname|port": { + "default_category": "Network activity", + "to_ids": 1 + }, + "mac-address": { + "default_category": "Network activity", + "to_ids": 0 + }, + "mac-eui-64": { + "default_category": "Network activity", + "to_ids": 0 + }, + "email-dst-display-name": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "email-src-display-name": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "email-header": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "email-reply-to": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "email-x-mailer": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "email-mime-boundary": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "email-thread-index": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "email-message-id": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "github-username": { + "default_category": "Social network", + "to_ids": 0 + }, + "github-repository": { + "default_category": "Social network", + "to_ids": 0 + }, + "github-organisation": { + "default_category": "Social network", + "to_ids": 0 + }, + "jabber-id": { + "default_category": "Social network", + "to_ids": 0 + }, + "twitter-id": { + "default_category": "Social network", + "to_ids": 0 + }, + "first-name": { + "default_category": "Person", + "to_ids": 0 + }, + "middle-name": { + "default_category": "Person", + "to_ids": 0 + }, + "last-name": { + "default_category": "Person", + "to_ids": 0 + }, + "date-of-birth": { + "default_category": "Person", + "to_ids": 0 + }, + "place-of-birth": { + "default_category": "Person", + "to_ids": 0 + }, + "gender": { + "default_category": "Person", + "to_ids": 0 + }, + "passport-number": { + "default_category": "Person", + "to_ids": 0 + }, + "passport-country": { + "default_category": "Person", + "to_ids": 0 + }, + "passport-expiration": { + "default_category": "Person", + "to_ids": 0 + }, + "redress-number": { + "default_category": "Person", + "to_ids": 0 + }, + "nationality": { + "default_category": "Person", + "to_ids": 0 + }, + "visa-number": { + "default_category": "Person", + "to_ids": 0 + }, + "issue-date-of-the-visa": { + "default_category": "Person", + "to_ids": 0 + }, + "primary-residence": { + "default_category": "Person", + "to_ids": 0 + }, + "country-of-residence": { + "default_category": "Person", + "to_ids": 0 + }, + "special-service-request": { + "default_category": "Person", + "to_ids": 0 + }, + "frequent-flyer-number": { + "default_category": "Person", + "to_ids": 0 + }, + "travel-details": { + "default_category": "Person", + "to_ids": 0 + }, + "payment-details": { + "default_category": "Person", + "to_ids": 0 + }, + "place-port-of-original-embarkation": { + "default_category": "Person", + "to_ids": 0 + }, + "place-port-of-clearance": { + "default_category": "Person", + "to_ids": 0 + }, + "place-port-of-onward-foreign-destination": { + "default_category": "Person", + "to_ids": 0 + }, + "passenger-name-record-locator-number": { + "default_category": "Person", + "to_ids": 0 + }, + "mobile-application-id": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "cortex": { + "default_category": "External analysis", + "to_ids": 0 + }, + "boolean": { + "default_category": "Other", + "to_ids": 0 } }, "types": [ - "AS", - "aba-rtn", - "attachment", - "authentihash", - "bank-account-nr", - "bic", - "bin", - "boolean", - "bro", - "btc", - "campaign-id", - "campaign-name", - "cc-number", - "cdhash", - "comment", - "cookie", - "cortex", - "counter", - "country-of-residence", - "cpe", - "date-of-birth", - "datetime", - "dns-soa-email", + "md5", + "sha1", + "sha256", + "filename", + "pdb", + "filename|md5", + "filename|sha1", + "filename|sha256", + "ip-src", + "ip-dst", + "hostname", "domain", "domain|ip", + "email-src", + "email-dst", + "email-subject", "email-attachment", "email-body", - "email-dst", - "email-dst-display-name", - "email-header", - "email-message-id", - "email-mime-boundary", - "email-reply-to", - "email-src", - "email-src-display-name", - "email-subject", - "email-thread-index", - "email-x-mailer", - "filename", - "filename|authentihash", - "filename|impfuzzy", - "filename|imphash", - "filename|md5", - "filename|pehash", - "filename|sha1", - "filename|sha224", - "filename|sha256", - "filename|sha384", - "filename|sha512", - "filename|sha512/224", - "filename|sha512/256", - "filename|ssdeep", - "filename|tlsh", - "first-name", "float", - "frequent-flyer-number", - "gender", - "gene", - "github-organisation", - "github-repository", - "github-username", - "hex", - "hostname", - "hostname|port", + "url", "http-method", - "iban", - "identity-card-number", - "impfuzzy", - "imphash", - "ip-dst", - "ip-dst|port", - "ip-src", - "ip-src|port", - "issue-date-of-the-visa", + "user-agent", "ja3-fingerprint-md5", - "jabber-id", - "last-name", - "link", - "mac-address", - "mac-eui-64", - "malware-sample", - "malware-type", - "md5", - "middle-name", - "mime-type", - "mobile-application-id", - "mutex", - "named pipe", - "nationality", - "other", - "passenger-name-record-locator-number", - "passport-country", - "passport-expiration", - "passport-number", - "pattern-in-file", - "pattern-in-memory", - "pattern-in-traffic", - "payment-details", - "pdb", - "pehash", - "phone-number", - "place-of-birth", - "place-port-of-clearance", - "place-port-of-onward-foreign-destination", - "place-port-of-original-embarkation", - "port", - "primary-residence", - "prtn", - "redress-number", "regkey", "regkey|value", - "sha1", + "AS", + "snort", + "bro", + "pattern-in-file", + "pattern-in-traffic", + "pattern-in-memory", + "yara", + "stix2-pattern", + "sigma", + "gene", + "mime-type", + "identity-card-number", + "cookie", + "vulnerability", + "attachment", + "malware-sample", + "link", + "comment", + "text", + "hex", + "other", + "named pipe", + "mutex", + "target-user", + "target-email", + "target-machine", + "target-org", + "target-location", + "target-external", + "btc", + "xmr", + "iban", + "bic", + "bank-account-nr", + "aba-rtn", + "bin", + "cc-number", + "prtn", + "phone-number", + "threat-actor", + "campaign-name", + "campaign-id", + "malware-type", + "uri", + "authentihash", + "ssdeep", + "imphash", + "pehash", + "impfuzzy", "sha224", - "sha256", "sha384", "sha512", "sha512/224", "sha512/256", - "sigma", - "size-in-bytes", - "snort", - "special-service-request", - "ssdeep", - "stix2-pattern", - "target-email", - "target-external", - "target-location", - "target-machine", - "target-org", - "target-user", - "text", - "threat-actor", "tlsh", - "travel-details", - "twitter-id", - "uri", - "url", - "user-agent", - "visa-number", - "vulnerability", - "whois-creation-date", + "cdhash", + "filename|authentihash", + "filename|ssdeep", + "filename|imphash", + "filename|impfuzzy", + "filename|pehash", + "filename|sha224", + "filename|sha384", + "filename|sha512", + "filename|sha512/224", + "filename|sha512/256", + "filename|tlsh", + "windows-scheduled-task", + "windows-service-name", + "windows-service-displayname", "whois-registrant-email", + "whois-registrant-phone", "whois-registrant-name", "whois-registrant-org", - "whois-registrant-phone", "whois-registrar", - "windows-scheduled-task", - "windows-service-displayname", - "windows-service-name", - "x509-fingerprint-md5", + "whois-creation-date", "x509-fingerprint-sha1", + "x509-fingerprint-md5", "x509-fingerprint-sha256", - "xmr", - "yara" - ] + "dns-soa-email", + "size-in-bytes", + "counter", + "datetime", + "cpe", + "port", + "ip-dst|port", + "ip-src|port", + "hostname|port", + "mac-address", + "mac-eui-64", + "email-dst-display-name", + "email-src-display-name", + "email-header", + "email-reply-to", + "email-x-mailer", + "email-mime-boundary", + "email-thread-index", + "email-message-id", + "github-username", + "github-repository", + "github-organisation", + "jabber-id", + "twitter-id", + "first-name", + "middle-name", + "last-name", + "date-of-birth", + "place-of-birth", + "gender", + "passport-number", + "passport-country", + "passport-expiration", + "redress-number", + "nationality", + "visa-number", + "issue-date-of-the-visa", + "primary-residence", + "country-of-residence", + "special-service-request", + "frequent-flyer-number", + "travel-details", + "payment-details", + "place-port-of-original-embarkation", + "place-port-of-clearance", + "place-port-of-onward-foreign-destination", + "passenger-name-record-locator-number", + "mobile-application-id", + "cortex", + "boolean" + ], + "categories": [ + "Internal reference", + "Targeting data", + "Antivirus detection", + "Payload delivery", + "Artifacts dropped", + "Payload installation", + "Persistence mechanism", + "Network activity", + "Payload type", + "Attribution", + "External analysis", + "Financial fraud", + "Support Tool", + "Social network", + "Person", + "Other" + ], + "category_type_mappings": { + "Internal reference": [ + "text", + "link", + "comment", + "other", + "hex" + ], + "Targeting data": [ + "target-user", + "target-email", + "target-machine", + "target-org", + "target-location", + "target-external", + "comment" + ], + "Antivirus detection": [ + "link", + "comment", + "text", + "hex", + "attachment", + "other" + ], + "Payload delivery": [ + "md5", + "sha1", + "sha224", + "sha256", + "sha384", + "sha512", + "sha512/224", + "sha512/256", + "ssdeep", + "imphash", + "impfuzzy", + "authentihash", + "pehash", + "tlsh", + "cdhash", + "filename", + "filename|md5", + "filename|sha1", + "filename|sha224", + "filename|sha256", + "filename|sha384", + "filename|sha512", + "filename|sha512/224", + "filename|sha512/256", + "filename|authentihash", + "filename|ssdeep", + "filename|tlsh", + "filename|imphash", + "filename|impfuzzy", + "filename|pehash", + "mac-address", + "mac-eui-64", + "ip-src", + "ip-dst", + "ip-dst|port", + "ip-src|port", + "hostname", + "domain", + "email-src", + "email-dst", + "email-subject", + "email-attachment", + "email-body", + "url", + "user-agent", + "AS", + "pattern-in-file", + "pattern-in-traffic", + "stix2-pattern", + "yara", + "sigma", + "mime-type", + "attachment", + "malware-sample", + "link", + "malware-type", + "comment", + "text", + "hex", + "vulnerability", + "x509-fingerprint-sha1", + "x509-fingerprint-md5", + "x509-fingerprint-sha256", + "ja3-fingerprint-md5", + "other", + "hostname|port", + "email-dst-display-name", + "email-src-display-name", + "email-header", + "email-reply-to", + "email-x-mailer", + "email-mime-boundary", + "email-thread-index", + "email-message-id", + "mobile-application-id", + "whois-registrant-email" + ], + "Artifacts dropped": [ + "md5", + "sha1", + "sha224", + "sha256", + "sha384", + "sha512", + "sha512/224", + "sha512/256", + "ssdeep", + "imphash", + "impfuzzy", + "authentihash", + "cdhash", + "filename", + "filename|md5", + "filename|sha1", + "filename|sha224", + "filename|sha256", + "filename|sha384", + "filename|sha512", + "filename|sha512/224", + "filename|sha512/256", + "filename|authentihash", + "filename|ssdeep", + "filename|tlsh", + "filename|imphash", + "filename|impfuzzy", + "filename|pehash", + "regkey", + "regkey|value", + "pattern-in-file", + "pattern-in-memory", + "pdb", + "stix2-pattern", + "yara", + "sigma", + "attachment", + "malware-sample", + "named pipe", + "mutex", + "windows-scheduled-task", + "windows-service-name", + "windows-service-displayname", + "comment", + "text", + "hex", + "x509-fingerprint-sha1", + "x509-fingerprint-md5", + "x509-fingerprint-sha256", + "other", + "cookie", + "gene", + "mime-type" + ], + "Payload installation": [ + "md5", + "sha1", + "sha224", + "sha256", + "sha384", + "sha512", + "sha512/224", + "sha512/256", + "ssdeep", + "imphash", + "impfuzzy", + "authentihash", + "pehash", + "tlsh", + "cdhash", + "filename", + "filename|md5", + "filename|sha1", + "filename|sha224", + "filename|sha256", + "filename|sha384", + "filename|sha512", + "filename|sha512/224", + "filename|sha512/256", + "filename|authentihash", + "filename|ssdeep", + "filename|tlsh", + "filename|imphash", + "filename|impfuzzy", + "filename|pehash", + "pattern-in-file", + "pattern-in-traffic", + "pattern-in-memory", + "stix2-pattern", + "yara", + "sigma", + "vulnerability", + "attachment", + "malware-sample", + "malware-type", + "comment", + "text", + "hex", + "x509-fingerprint-sha1", + "x509-fingerprint-md5", + "x509-fingerprint-sha256", + "mobile-application-id", + "other", + "mime-type" + ], + "Persistence mechanism": [ + "filename", + "regkey", + "regkey|value", + "comment", + "text", + "other", + "hex" + ], + "Network activity": [ + "ip-src", + "ip-dst", + "ip-dst|port", + "ip-src|port", + "port", + "hostname", + "domain", + "domain|ip", + "mac-address", + "mac-eui-64", + "email-dst", + "url", + "uri", + "user-agent", + "http-method", + "AS", + "snort", + "pattern-in-file", + "stix2-pattern", + "pattern-in-traffic", + "attachment", + "comment", + "text", + "x509-fingerprint-md5", + "x509-fingerprint-sha1", + "x509-fingerprint-sha256", + "ja3-fingerprint-md5", + "other", + "hex", + "cookie", + "hostname|port", + "bro" + ], + "Payload type": [ + "comment", + "text", + "other" + ], + "Attribution": [ + "threat-actor", + "campaign-name", + "campaign-id", + "whois-registrant-phone", + "whois-registrant-email", + "whois-registrant-name", + "whois-registrant-org", + "whois-registrar", + "whois-creation-date", + "comment", + "text", + "x509-fingerprint-sha1", + "x509-fingerprint-md5", + "x509-fingerprint-sha256", + "other", + "dns-soa-email" + ], + "External analysis": [ + "md5", + "sha1", + "sha256", + "filename", + "filename|md5", + "filename|sha1", + "filename|sha256", + "ip-src", + "ip-dst", + "ip-dst|port", + "ip-src|port", + "mac-address", + "mac-eui-64", + "hostname", + "domain", + "domain|ip", + "url", + "user-agent", + "regkey", + "regkey|value", + "AS", + "snort", + "bro", + "pattern-in-file", + "pattern-in-traffic", + "pattern-in-memory", + "vulnerability", + "attachment", + "malware-sample", + "link", + "comment", + "text", + "x509-fingerprint-sha1", + "x509-fingerprint-md5", + "x509-fingerprint-sha256", + "ja3-fingerprint-md5", + "github-repository", + "other", + "cortex" + ], + "Financial fraud": [ + "btc", + "xmr", + "iban", + "bic", + "bank-account-nr", + "aba-rtn", + "bin", + "cc-number", + "prtn", + "phone-number", + "comment", + "text", + "other", + "hex" + ], + "Support Tool": [ + "link", + "text", + "attachment", + "comment", + "other", + "hex" + ], + "Social network": [ + "github-username", + "github-repository", + "github-organisation", + "jabber-id", + "twitter-id", + "email-src", + "email-dst", + "comment", + "text", + "other", + "whois-registrant-email" + ], + "Person": [ + "first-name", + "middle-name", + "last-name", + "date-of-birth", + "place-of-birth", + "gender", + "passport-number", + "passport-country", + "passport-expiration", + "redress-number", + "nationality", + "visa-number", + "issue-date-of-the-visa", + "primary-residence", + "country-of-residence", + "special-service-request", + "frequent-flyer-number", + "travel-details", + "payment-details", + "place-port-of-original-embarkation", + "place-port-of-clearance", + "place-port-of-onward-foreign-destination", + "passenger-name-record-locator-number", + "comment", + "text", + "other", + "phone-number", + "identity-card-number" + ], + "Other": [ + "comment", + "text", + "other", + "size-in-bytes", + "counter", + "datetime", + "cpe", + "port", + "float", + "hex", + "phone-number", + "boolean" + ] + } } } From 2c882c1887807ef8c8462f582415470448e5d68c Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Sun, 30 Dec 2018 16:18:27 +0100 Subject: [PATCH 33/38] chg: [misp-objects] templates updated to the latest version --- pymisp/data/misp-objects | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index 11a462e..b659345 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit 11a462e79b02428a08b11698d45aa8aa5ab6887d +Subproject commit b6593451c2eb7765246e94cb88b650f2a65428ce From b9d865b756c5fa35199f52a5ce0d33519e8f8a3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 3 Jan 2019 11:48:49 +0100 Subject: [PATCH 34/38] fix: Use new API in get_csv.py Fix #314 --- examples/get_csv.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/examples/get_csv.py b/examples/get_csv.py index 33baf62..5921e53 100755 --- a/examples/get_csv.py +++ b/examples/get_csv.py @@ -1,9 +1,9 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- import argparse -from pymisp import PyMISP +from pymisp import ExpandedPyMISP from keys import misp_url, misp_key, misp_verifycert @@ -14,12 +14,20 @@ if __name__ == '__main__': parser.add_argument("-o", "--object_attribute", nargs='+', help="Object attribute column names") parser.add_argument("-t", "--misp_types", nargs='+', help="MISP types to fetch (ip-src, hostname, ...)") parser.add_argument("-c", "--context", action='store_true', help="Add event level context (tags...)") - parser.add_argument("-i", "--ignore", action='store_true', help="Returns the attributes even if the event isn't published, or the attribute doesn't have the to_ids flag") parser.add_argument("-f", "--outfile", help="Output file to write the CSV.") args = parser.parse_args() - pymisp = PyMISP(misp_url, misp_key, misp_verifycert, debug=True) - response = pymisp.get_csv(args.event_id, args.attribute, args.object_attribute, args.misp_types, args.context, args.ignore) + pymisp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert, debug=True) + attr = [] + if args.attribute: + attr += args.attribute + if args.object_attribute: + attr += args.object_attribute + if not attr: + attr = None + print(args.context) + response = pymisp.search(return_format='csv', eventid=args.event_id, requested_attributes=attr, + type_attribute=args.misp_types, include_context=args.context) if args.outfile: with open(args.outfile, 'w') as f: From 13ec75df29dd4b81c6af4dc776d33ab42a1f9031 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Sun, 13 Jan 2019 11:55:00 +0100 Subject: [PATCH 35/38] chg: [data] new types added (hassh-md5 and hasshserver-md5) --- pymisp/data/describeTypes.json | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pymisp/data/describeTypes.json b/pymisp/data/describeTypes.json index 5f2bec3..3019122 100644 --- a/pymisp/data/describeTypes.json +++ b/pymisp/data/describeTypes.json @@ -93,6 +93,14 @@ "default_category": "Network activity", "to_ids": 1 }, + "hashh-md5": { + "default_category": "Network activity", + "to_ids": 1 + }, + "hashhserver-md5": { + "default_category": "Network activity", + "to_ids": 1 + }, "regkey": { "default_category": "Persistence mechanism", "to_ids": 1 @@ -642,6 +650,8 @@ "http-method", "user-agent", "ja3-fingerprint-md5", + "hashh-md5", + "hashhserver-md5", "regkey", "regkey|value", "AS", @@ -882,6 +892,8 @@ "x509-fingerprint-md5", "x509-fingerprint-sha256", "ja3-fingerprint-md5", + "hassh-md5", + "hasshserver-md5", "other", "hostname|port", "email-dst-display-name", @@ -1038,6 +1050,8 @@ "x509-fingerprint-sha1", "x509-fingerprint-sha256", "ja3-fingerprint-md5", + "hassh-md5", + "hasshserver-md5", "other", "hex", "cookie", @@ -1104,6 +1118,8 @@ "x509-fingerprint-md5", "x509-fingerprint-sha256", "ja3-fingerprint-md5", + "hassh-md5", + "hasshserver-md5", "github-repository", "other", "cortex" From f5e621b2a52ee8ab6b0860e4b7d09169be5e1d69 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Sun, 13 Jan 2019 12:07:24 +0100 Subject: [PATCH 36/38] chg: [data] fix describeTypes --- pymisp/data/describeTypes.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pymisp/data/describeTypes.json b/pymisp/data/describeTypes.json index 3019122..015b943 100644 --- a/pymisp/data/describeTypes.json +++ b/pymisp/data/describeTypes.json @@ -93,11 +93,11 @@ "default_category": "Network activity", "to_ids": 1 }, - "hashh-md5": { + "hassh-md5": { "default_category": "Network activity", "to_ids": 1 }, - "hashhserver-md5": { + "hasshserver-md5": { "default_category": "Network activity", "to_ids": 1 }, @@ -650,8 +650,8 @@ "http-method", "user-agent", "ja3-fingerprint-md5", - "hashh-md5", - "hashhserver-md5", + "hassh-md5", + "hasshserver-md5", "regkey", "regkey|value", "AS", From ee86f6fa447004b9dc1d5021eb495037291e1d8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 14 Jan 2019 10:11:58 +0100 Subject: [PATCH 37/38] fix: The wrong class name was used when there is an error at Event creation. --- pymisp/mispevent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 18cd938..ff7ad33 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -488,14 +488,14 @@ class MISPEvent(AbstractMISP): # Required value self.info = kwargs.pop('info', None) if self.info is None: - raise NewAttributeError('The info field of the new event is required.') + raise NewEventError('The info field of the new event is required.') # Default values for a valid event to send to a MISP instance self.distribution = kwargs.pop('distribution', None) if self.distribution is not None: self.distribution = int(self.distribution) if self.distribution not in [0, 1, 2, 3, 4]: - raise NewAttributeError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 4'.format(self.distribution)) + raise NewEventError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 4'.format(self.distribution)) if kwargs.get('threat_level_id') is not None: self.threat_level_id = int(kwargs.pop('threat_level_id')) From 4687cb80349387e7ce38b25e0c9aaa684652785a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 14 Jan 2019 10:12:43 +0100 Subject: [PATCH 38/38] chg: Bump misp-objects --- pymisp/data/misp-objects | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index b659345..b25388c 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit b6593451c2eb7765246e94cb88b650f2a65428ce +Subproject commit b25388c406e2f8dfcba94fe742a3ca90360315fd