From 31b1b932e5da26535b63f8c943ba4e5c5dc2332f Mon Sep 17 00:00:00 2001 From: c-goes Date: Fri, 6 Oct 2017 13:08:28 +0200 Subject: [PATCH 001/125] added default_category for email-message-id --- pymisp/data/describeTypes.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/data/describeTypes.json b/pymisp/data/describeTypes.json index 6435855..bff6ac3 100644 --- a/pymisp/data/describeTypes.json +++ b/pymisp/data/describeTypes.json @@ -438,7 +438,7 @@ "to_ids": 0 }, "email-message-id": { - "default_category": "", + "default_category": "Payload delivery", "to_ids": 0 }, "github-username": { From 78eb6e3080d1314bdaaa187a81f9b0e3d71f3dc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 25 Oct 2017 15:00:00 -0400 Subject: [PATCH 002/125] fix: Update dependencies for VT generator. --- .travis.yml | 2 +- pymisp/tools/vtreportobject.py | 9 ++++++++- setup.py | 3 ++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index e6a5ba7..93ef5eb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,7 @@ install: - pip install -U nose pip setuptools - pip install coveralls codecov requests-mock - pip install git+https://github.com/kbandla/pydeep.git - - pip install .[fileobjects,neo,openioc] + - pip install .[fileobjects,neo,openioc,virustotal] - pushd tests - git clone https://github.com/viper-framework/viper-test-files.git - popd diff --git a/pymisp/tools/vtreportobject.py b/pymisp/tools/vtreportobject.py index 5a4858b..927ca26 100644 --- a/pymisp/tools/vtreportobject.py +++ b/pymisp/tools/vtreportobject.py @@ -4,7 +4,12 @@ import re import requests -import validators +try: + import validators + has_validators = True +except ImportError: + has_validators = False + from .abstractgenerator import AbstractMISPObjectGenerator from .. import InvalidMISPObject @@ -48,6 +53,8 @@ class VTReportObject(AbstractMISPObjectGenerator): :ioc: Indicator to search VirusTotal for ''' + if not has_validators: + raise Exception('You need to install validators: pip install validators') if validators.url(ioc): return "url" elif re.match(r"\b([a-fA-F0-9]{32}|[a-fA-F0-9]{40}|[a-fA-F0-9]{64})\b", ioc): diff --git a/setup.py b/setup.py index 528b2d6..5d7391d 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,8 @@ setup( install_requires=['six', 'requests', 'python-dateutil', 'jsonschema', 'setuptools>=36.4'], extras_require={'fileobjects': ['lief>=0.8', 'python-magic'], 'neo': ['py2neo'], - 'openioc': ['beautifulsoup4']}, + 'openioc': ['beautifulsoup4'], + 'virustotal': ['validators']}, tests_require=[ 'jsonschema', 'python-dateutil', From 14bc9e4b19199b695c84e97a5baa7ea64bbfd6a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 26 Oct 2017 12:05:51 -0400 Subject: [PATCH 003/125] fix: Properly pop the distribution key. --- pymisp/mispevent.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index f00c5cf..3f99d7a 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -177,8 +177,9 @@ class MISPAttribute(AbstractMISP): if not isinstance(self.to_ids, bool): raise NewAttributeError('{} is invalid, to_ids has to be True or False'.format(self.to_ids)) - if kwargs.get('distribution') is not None: - self.distribution = int(kwargs.pop('distribution')) + self.distribution = kwargs.pop('distribution', None) + if self.distribution is not None: + self.distribution = int(self.distribution) if self.distribution not in [0, 1, 2, 3, 4, 5]: raise NewAttributeError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 4, 5'.format(self.distribution)) From 30da658292e9671f0c32bfe28bcebacb04b59540 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sat, 28 Oct 2017 16:57:03 -0400 Subject: [PATCH 004/125] chg: Remove warning if PyMISP is too new --- pymisp/api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index adc9784..fde42d8 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -118,9 +118,9 @@ class PyMISP(object): else: pymisp_version_tup = tuple(int(x) for x in __version__.split('.')) recommended_version_tup = tuple(int(x) for x in response['version'].split('.')) - if recommended_version_tup < pymisp_version_tup: - logger.warning("The version of PyMISP recommended by the MISP instance ({}) is older than the one you're using now ({}). Please upgrade the MISP instance or use an older PyMISP version.".format(response['version'], __version__)) - elif pymisp_version_tup < recommended_version_tup: + if recommended_version_tup < pymisp_version_tup[:3]: + logger.info("The version of PyMISP recommended by the MISP instance ({}) is older than the one you're using now ({}). If you have a problem, please upgrade the MISP instance or use an older PyMISP version.".format(response['version'], __version__)) + elif pymisp_version_tup[:3] < recommended_version_tup: logger.warning("The version of PyMISP recommended by the MISP instance ({}) is newer than the one you're using now ({}). Please upgrade PyMISP.".format(response['version'], __version__)) except Exception as e: From ea327ceffb1d19aeba027498270528e03b89b64c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sat, 28 Oct 2017 16:58:50 -0400 Subject: [PATCH 005/125] chg: Update asciidoctor generator --- examples/asciidoc_generator.py | 115 +++++++++++------------------ examples/profiles/__init__.py | 0 examples/profiles/daily_report.py | 37 ++++++++++ examples/profiles/weekly_report.py | 33 +++++++++ 4 files changed, 115 insertions(+), 70 deletions(-) create mode 100644 examples/profiles/__init__.py create mode 100644 examples/profiles/daily_report.py create mode 100644 examples/profiles/weekly_report.py diff --git a/examples/asciidoc_generator.py b/examples/asciidoc_generator.py index 156a7ff..f1a09a5 100755 --- a/examples/asciidoc_generator.py +++ b/examples/asciidoc_generator.py @@ -1,55 +1,21 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +import argparse +from datetime import date +import importlib + from pymisp import MISPEvent from defang import defang -import argparse from pytaxonomies import Taxonomies -from datetime import date -headers = """ -:toc: right -:toclevels: 1 -:toc-title: Daily Report -:icons: font -:sectanchors: -:sectlinks: -= Daily report by {org_name} -{date} - -:icons: font - -""" - -event_level_tags = """ -IMPORTANT: This event is classified TLP:{value}. - -{expanded} - -""" - -attributes = """ -=== Indicator(s) of compromise - -{list_attributes} - -""" - -title = """ -== ({internal_id}) {title} - -{summary} - -""" - -types_to_attach = ['ip-dst', 'url', 'domain'] -objects_to_attach = ['domain-ip'] class ReportGenerator(): - - def __init__(self): + def __init__(self, profile="daily_report"): self.taxonomies = Taxonomies() self.report = '' + profile_name = "profiles.{}".format(profile) + self.template = importlib.import_module(name=profile_name) def from_remote(self, event_id): from pymisp import PyMISP @@ -66,15 +32,16 @@ class ReportGenerator(): def attributes(self): if not self.misp_event.attributes: return '' - list_attributes = '' + list_attributes = [] for attribute in self.misp_event.attributes: - if attribute.type in types_to_attach: - list_attributes += "\n* {}\n".format(defang(attribute.value)) + if attribute.type in self.template.types_to_attach: + list_attributes.append("* {}".format(defang(attribute.value))) for obj in self.misp_event.Object: - for attribute in obj.Attribute: - if attribute.type in types_to_attach: - list_attributes += "\n* {}\n".format(defang(attribute.value)) - return attributes.format(list_attributes=list_attributes) + if obj.name in self.template.objects_to_attach: + for attribute in obj.Attribute: + if attribute.type in self.template.types_to_attach: + list_attributes.append("* {}".format(defang(attribute.value))) + return self.template.attributes.format(list_attributes="\n".join(list_attributes)) def _get_tag_info(self, machinetag): return self.taxonomies.revert_machinetag(machinetag) @@ -82,7 +49,7 @@ class ReportGenerator(): def report_headers(self): content = {'org_name': 'name', 'date': date.today().isoformat()} - self.report += headers.format(**content) + self.report += self.template.headers.format(**content) def event_level_tags(self): if not self.misp_event.Tag: @@ -91,7 +58,7 @@ class ReportGenerator(): # Only look for TLP for now if tag['name'].startswith('tlp'): tax, predicate = self._get_tag_info(tag['name']) - return event_level_tags.format(value=predicate.predicate.upper(), expanded=predicate.expanded) + return self.template.event_level_tags.format(value=predicate.predicate.upper(), expanded=predicate.expanded) def title(self): internal_id = '' @@ -106,34 +73,42 @@ class ReportGenerator(): if a.object_relation == 'summary': summary = a.value - return title.format(internal_id=internal_id, title=self.misp_event.info, - summary=summary) - + return self.template.title.format(internal_id=internal_id, title=self.misp_event.info, + summary=summary) def asciidoc(self, lang='en'): self.report += self.title() self.report += self.event_level_tags() self.report += self.attributes() + if __name__ == '__main__': + try: + parser = argparse.ArgumentParser(description='Create a human-readable report out of a MISP event') + parser.add_argument("--profile", default="daily_report", help="Profile template to use") + parser.add_argument("-o", "--output", help="Output file to write to (generally ends in .adoc)") + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("-e", "--event", default=[], nargs='+', help="Event ID to get.") + group.add_argument("-p", "--path", default=[], nargs='+', help="Path to the JSON dump.") - parser = argparse.ArgumentParser(description='Create a human-readable report out of a MISP event') - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument("-e", "--event", default=[], nargs='+', help="Event ID to get.") - group.add_argument("-p", "--path", default=[], nargs='+', help="Path to the JSON dump.") + args = parser.parse_args() - args = parser.parse_args() + report = ReportGenerator(args.profile) + report.report_headers() - report = ReportGenerator() - report.report_headers() + if args.event: + for eid in args.event: + report.from_remote(eid) + report.asciidoc() + else: + for f in args.path: + report.from_file(f) + report.asciidoc() - if args.event: - for eid in args.event: - report.from_remote(eid) - report.asciidoc() - else: - for f in args.path: - report.from_file(f) - report.asciidoc() - - print(report.report) + if args.output: + with open(args.output, "w") as ofile: + ofile.write(report.report) + else: + print(report.report) + except ModuleNotFoundError as err: + print(err) diff --git a/examples/profiles/__init__.py b/examples/profiles/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/examples/profiles/daily_report.py b/examples/profiles/daily_report.py new file mode 100644 index 0000000..2799333 --- /dev/null +++ b/examples/profiles/daily_report.py @@ -0,0 +1,37 @@ +types_to_attach = ['ip-dst', 'url', 'domain'] +objects_to_attach = ['domain-ip'] + +headers = """ +:toc: right +:toclevels: 1 +:toc-title: Daily Report +:icons: font +:sectanchors: +:sectlinks: += Daily report by {org_name} +{date} + +:icons: font + +""" + +event_level_tags = """ +IMPORTANT: This event is classified TLP:{value}. + +{expanded} + +""" + +attributes = """ +=== Indicator(s) of compromise + +{list_attributes} + +""" + +title = """ +== ({internal_id}) {title} + +{summary} + +""" diff --git a/examples/profiles/weekly_report.py b/examples/profiles/weekly_report.py new file mode 100644 index 0000000..1005fd9 --- /dev/null +++ b/examples/profiles/weekly_report.py @@ -0,0 +1,33 @@ +types_to_attach = ['ip-dst', 'url', 'domain', 'md5'] +objects_to_attach = ['domain-ip', 'file'] + +headers = """ +:toc: right +:toclevels: 1 +:toc-title: Weekly Report +:icons: font +:sectanchors: +:sectlinks: += Weekly report by {org_name} +{date} + +:icons: font + +""" + +event_level_tags = """ +""" + +attributes = """ +=== Indicator(s) of compromise + +{list_attributes} + +""" + +title = """ +== ({internal_id}) {title} + +{summary} + +""" From 7ece6b7fbcc2be9e645f52acd804abd63689410f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sat, 28 Oct 2017 17:09:11 -0400 Subject: [PATCH 006/125] fix: Properly set the distribution at event level fix #120 --- pymisp/api.py | 2 +- pymisp/mispevent.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index fde42d8..0069153 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -265,7 +265,7 @@ class PyMISP(object): misp_event.publish() return misp_event - def _prepare_full_attribute(self, category, type_value, value, to_ids, comment=None, distribution=5, **kwargs): + def _prepare_full_attribute(self, category, type_value, value, to_ids, comment=None, distribution=None, **kwargs): """Initialize a new MISPAttribute from scratch""" misp_attribute = MISPAttribute(self.describe_types) misp_attribute.set_all_values(type=type_value, value=value, category=category, diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 3f99d7a..130d905 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -428,8 +428,9 @@ class MISPEvent(AbstractMISP): raise NewAttributeError('The info field of the new event is required.') # Default values for a valid event to send to a MISP instance - if kwargs.get('distribution') is not None: - self.distribution = int(kwargs.pop('distribution')) + self.distribution = kwargs.pop('distribution', None) + if self.distribution is not None: + self.distribution = int(self.distribution) if self.distribution not in [0, 1, 2, 3, 4]: raise NewAttributeError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 4'.format(self.distribution)) From 4b53b399d0efb87ecb7a8f55ba78314ceeecc820 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 1 Nov 2017 17:15:49 -0700 Subject: [PATCH 007/125] fix: Properly upload a sample in an existing event. Fix https://github.com/MISP/PyMISP/issues/123 --- pymisp/api.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 0069153..2272877 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -759,8 +759,6 @@ class PyMISP(object): to_post['request']['info'] = misp_event.info to_post['request']['analysis'] = misp_event.analysis to_post['request']['threat_level_id'] = misp_event.threat_level_id - else: - to_post['request']['event_id'] = int(event_id) default_values = self.sane_default['malware-sample'] if to_ids is None or not isinstance(to_ids, bool): @@ -772,7 +770,7 @@ class PyMISP(object): to_post['request']['category'] = category to_post['request']['comment'] = comment - return to_post + return to_post, event_id def _encode_file_to_upload(self, filepath_or_bytes): """Helper to encode a file to upload""" @@ -787,29 +785,33 @@ class PyMISP(object): to_ids=True, category=None, comment=None, info=None, analysis=None, threat_level_id=None): """Upload a sample""" - to_post = self._prepare_upload(event_id, distribution, to_ids, category, - comment, info, analysis, threat_level_id) + to_post, event_id = self._prepare_upload(event_id, distribution, to_ids, category, + comment, info, analysis, threat_level_id) to_post['request']['files'] = [{'filename': filename, 'data': self._encode_file_to_upload(filepath_or_bytes)}] - return self._upload_sample(to_post) + return self._upload_sample(to_post, event_id) def upload_samplelist(self, filepaths, event_id, distribution=None, to_ids=True, category=None, comment=None, info=None, analysis=None, threat_level_id=None): """Upload a list of samples""" - to_post = self._prepare_upload(event_id, distribution, to_ids, category, - comment, info, analysis, threat_level_id) + to_post, event_id = self._prepare_upload(event_id, distribution, to_ids, category, + comment, info, analysis, threat_level_id) files = [] for path in filepaths: if not os.path.isfile(path): continue files.append({'filename': os.path.basename(path), 'data': self._encode_file_to_upload(path)}) to_post['request']['files'] = files - return self._upload_sample(to_post) + return self._upload_sample(to_post, event_id) - def _upload_sample(self, to_post): + def _upload_sample(self, to_post, event_id=None): """Helper to upload a sample""" session = self.__prepare_session() - url = urljoin(self.root_url, 'events/upload_sample') + if event_id is None: + url = urljoin(self.root_url, 'events/upload_sample') + else: + url = urljoin(self.root_url, 'events/upload_sample/{}'.format(event_id)) + logger.info(to_post) response = session.post(url, data=json.dumps(to_post)) return self._check_response(response) From e5a7153284b3fdcba3b135e2785023194e95d677 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 1 Nov 2017 17:26:58 -0700 Subject: [PATCH 008/125] fix: Properly pass the distribution when uploading a sample Fix: https://github.com/MISP/PyMISP/issues/129 --- pymisp/api.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pymisp/api.py b/pymisp/api.py index 2272877..bbbfe84 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -759,6 +759,9 @@ class PyMISP(object): to_post['request']['info'] = misp_event.info to_post['request']['analysis'] = misp_event.analysis to_post['request']['threat_level_id'] = misp_event.threat_level_id + else: + if distribution is not None: + to_post['request']['distribution'] = distribution default_values = self.sane_default['malware-sample'] if to_ids is None or not isinstance(to_ids, bool): From 75d96b2d7a89406997035a1715de7c3cbce2303b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 2 Nov 2017 09:26:05 -0700 Subject: [PATCH 009/125] fix: Allow to load non-malware ZIP files in MISP Event Prior to his patch, any zip file loaded by MISP Event was unpacked and processed as an excrypted malware from MISP. --- pymisp/mispevent.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 130d905..098dd4f 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -221,16 +221,33 @@ class MISPAttribute(AbstractMISP): self._malware_binary = self.data self.encrypt = True + def __is_misp_encrypted_file(self, f): + files_list = f.namelist() + if len(files_list) != 2: + return False + md5_from_filename = '' + md5_from_file = '' + for name in files_list: + if name.endswith('.filename.txt'): + md5_from_filename = name.replace('.filename.txt', '') + else: + md5_from_file = name + if not md5_from_filename or not md5_from_file or md5_from_filename != md5_from_file: + return False + return True + def _load_data(self): if not isinstance(self.data, BytesIO): self.data = BytesIO(base64.b64decode(self.data)) if self.type == 'malware-sample': try: with ZipFile(self.data) as f: + if not self.__is_misp_encrypted_file(f): + raise Exception('Not an existing malware sample') for name in f.namelist(): - if name.endswith('.txt'): + if name.endswith('.filename.txt'): with f.open(name, pwd=b'infected') as unpacked: - self.malware_filename = unpacked.read().decode() + self.malware_filename = unpacked.read().decode().strip() else: with f.open(name, pwd=b'infected') as unpacked: self._malware_binary = BytesIO(unpacked.read()) From 0e123af546b7de00f65ee65a4b72402c3d10dd11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 2 Nov 2017 09:57:53 -0700 Subject: [PATCH 010/125] fix: Let load unknown object relations in known templates This isn't recommended, but happens very often. --- pymisp/mispevent.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 098dd4f..20eeb49 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -721,7 +721,11 @@ class MISPObject(AbstractMISP): if value.get('value') is None: return None if self.__known_template: - attribute = MISPObjectAttribute(self.__definition['attributes'][object_relation]) + if self.__definition['attributes'].get(object_relation): + attribute = MISPObjectAttribute(self.__definition['attributes'][object_relation]) + else: + # Woopsie, this object_relation is unknown, no sane defaults for you. + attribute = MISPObjectAttribute({}) else: attribute = MISPObjectAttribute({}) attribute.from_dict(object_relation, **value) From 195cd6d7fc305ac6628ed8f2ff762b3f69a9b6ca Mon Sep 17 00:00:00 2001 From: iglocska Date: Sat, 4 Nov 2017 14:18:15 +0100 Subject: [PATCH 011/125] Rework of the feed generator - use objects, attribute tags and object references correctly - generate quickhashlist for fast lookups / future MISP caching mechanism - saner structure (herp-a-derp) --- examples/feed-generator/generate.py | 169 +++++++++++++++++++--------- 1 file changed, 113 insertions(+), 56 deletions(-) diff --git a/examples/feed-generator/generate.py b/examples/feed-generator/generate.py index 2188d2a..a2e4d3b 100755 --- a/examples/feed-generator/generate.py +++ b/examples/feed-generator/generate.py @@ -4,28 +4,79 @@ import sys import json import os +import hashlib from pymisp import PyMISP from settings import url, key, ssl, outputdir, filters, valid_attribute_distribution_levels +objectsFields = { + 'Attribute': { + 'uuid', + 'value', + 'category', + 'type', + 'comment', + 'data', + 'timestamp', + 'to_ids' + }, + 'Event': { + 'uuid', + 'info', + 'threat_level_id', + 'analysis', + 'timestamp', + 'publish_timestamp', + 'published', + 'date' + }, + 'Object': { + 'name', + 'meta-category', + 'description', + 'template_uuid', + 'template_version', + 'uuid', + 'timestamp', + 'distribution', + 'sharing_group_id', + 'comment' + }, + 'ObjectReference': { + 'uuid', + 'timestamp', + 'relationship_type', + 'comment', + 'object_uuid', + 'referenced_uuid' + }, + 'Orgc': { + 'name', + 'uuid' + }, + 'Tag': { + 'name', + 'colour', + 'exportable' + } +} -objectsToSave = {'Orgc': {'fields': ['name', 'uuid'], - 'multiple': False, - }, - 'Tag': {'fields': ['name', 'colour', 'exportable'], - 'multiple': True, - }, - 'Attribute': {'fields': ['uuid', 'value', 'category', 'type', - 'comment', 'data', 'timestamp', 'to_ids'], - 'multiple': True, - }, - } - -fieldsToSave = ['uuid', 'info', 'threat_level_id', 'analysis', - 'timestamp', 'publish_timestamp', 'published', - 'date'] +objectsToSave = { + 'Orgc': {}, + 'Tag': {}, + 'Attribute': { + 'Tag': {} + }, + 'Object': { + 'Attribute': { + 'Tag': {} + }, + 'ObjectReference': {} + } +} valid_attribute_distributions = [] +attributeHashes = [] def init(): # If we have an old settings.py file then this variable won't exist @@ -36,61 +87,65 @@ def init(): valid_attribute_distributions = ['0', '1', '2', '3', '4', '5'] return PyMISP(url, key, ssl) +def recursiveExtract(container, containerType, leaf, eventUuid): + temp = {} + if containerType in ['Attribute', 'Object']: + if (__blockByDistribution(container)): + return False + for field in objectsFields[containerType]: + if field in container: + temp[field] = container[field] + if (containerType == 'Attribute'): + global attributeHashes + if ('|' in container['type'] or container['type'] == 'malware-sample'): + split = container['value'].split('|') + attributeHashes.append([hashlib.md5(split[0].encode("utf-8")).hexdigest(), eventUuid]) + attributeHashes.append([hashlib.md5(split[1].encode("utf-8")).hexdigest(), eventUuid]) + else: + attributeHashes.append([hashlib.md5(container['value'].encode("utf-8")).hexdigest(), eventUuid]) + children = leaf.keys() + for childType in children: + childContainer = container.get(childType) + if (childContainer): + if (type(childContainer) is dict): + temp[childType] = recursiveExtract(childContainer, childType, leaf[childType], eventUuid) + else: + temp[childType] = [] + for element in childContainer: + processed = recursiveExtract(element, childType, leaf[childType], eventUuid) + if (processed): + temp[childType].append(processed) + return temp def saveEvent(misp, uuid): + result = {} event = misp.get_event(uuid) if not event.get('Event'): print('Error while fetching event: {}'.format(event['message'])) sys.exit('Could not create file for event ' + uuid + '.') - event = __cleanUpEvent(event) + event['Event'] = recursiveExtract(event['Event'], 'Event', objectsToSave, event['Event']['uuid']) event = json.dumps(event) eventFile = open(os.path.join(outputdir, uuid + '.json'), 'w') eventFile.write(event) eventFile.close() - -def __cleanUpEvent(event): - temp = event - event = {'Event': {}} - __cleanupEventFields(event, temp) - __cleanupEventObjects(event, temp) - return event - - -def __cleanupEventFields(event, temp): - for field in fieldsToSave: - if field in temp['Event'].keys(): - event['Event'][field] = temp['Event'][field] - return event - - -def __blockAttributeByDistribution(attribute): - if attribute['distribution'] not in valid_attribute_distributions: +def __blockByDistribution(element): + if element['distribution'] not in valid_attribute_distributions: return True return False +def saveHashes(): + if not attributeHashes: + return False + try: + hashFile = open(os.path.join(outputdir, 'hashes.csv'), 'w') + for element in attributeHashes: + hashFile.write('{},{}\n'.format(element[0], element[1])) + hashFile.close() + except Exception as e: + print(e) + sys.exit('Could not create the quick hash lookup file.') -def __cleanupEventObjects(event, temp): - for objectType in objectsToSave.keys(): - if objectsToSave[objectType]['multiple'] is True: - if objectType in temp['Event']: - for objectInstance in temp['Event'][objectType]: - if objectType is 'Attribute': - if __blockAttributeByDistribution(objectInstance): - continue - tempObject = {} - for field in objectsToSave[objectType]['fields']: - if field in objectInstance.keys(): - tempObject[field] = objectInstance[field] - if objectType not in event['Event']: - event['Event'][objectType] = [] - event['Event'][objectType].append(tempObject) - else: - tempObject = {} - for field in objectsToSave[objectType]['fields']: - tempObject[field] = temp['Event'][objectType][field] - event['Event'][objectType] = tempObject - return event def saveManifest(manifest): @@ -138,4 +193,6 @@ if __name__ == '__main__': print("Event " + str(counter) + "/" + str(total) + " exported.") counter += 1 saveManifest(manifest) - print('Manifest saved. Feed creation completed.') + print('Manifest saved.') + saveHashes() + print('Hashes saved. Feed creation completed.') From 134df0cafb8897da8798fb16e58ff4732c245f70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 7 Nov 2017 18:10:04 -0800 Subject: [PATCH 012/125] chg: Properly use python logging module. --- pymisp/__init__.py | 10 +++++-- pymisp/abstract.py | 6 +++-- pymisp/api.py | 42 ++++++++++++++---------------- pymisp/data/misp-objects | 2 +- pymisp/mispevent.py | 5 ++-- pymisp/tools/create_misp_object.py | 16 +++++++----- pymisp/tools/elfobject.py | 5 ++-- pymisp/tools/fileobject.py | 9 ++++--- pymisp/tools/machoobject.py | 6 +++-- pymisp/tools/peobject.py | 5 ++-- 10 files changed, 60 insertions(+), 46 deletions(-) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 55d2a19..3edf8b1 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,4 +1,9 @@ __version__ = '2.4.81.2' +import sys +import logging +logger = logging.getLogger(__name__) +FORMAT = "[%(filename)s:%(lineno)s - %(funcName)s() ] %(message)s" +logging.basicConfig(stream=sys.stderr, level=logging.WARNING, format=FORMAT) try: from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate # noqa @@ -9,5 +14,6 @@ try: from .tools import Neo4j # noqa from .tools import stix # noqa from .tools import openioc # noqa -except ImportError: - pass + logger.debug('pymisp loaded properly') +except ImportError as e: + logger.warning('Unable to load pymisp properly: {}'.format(e)) diff --git a/pymisp/abstract.py b/pymisp/abstract.py index a57795d..2a5d697 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -6,10 +6,12 @@ import json from json import JSONEncoder import collections import six # Remove that import when discarding python2 support. +import logging + +logger = logging.getLogger('pymisp') if six.PY2: - import warnings - warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5") + logger.warning("You're using python 2, it is strongly recommended to use python >=3.5") class MISPEncode(JSONEncoder): diff --git a/pymisp/api.py b/pymisp/api.py index bbbfe84..3ec4ed3 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -9,16 +9,19 @@ import datetime import os import base64 import re -import warnings import functools import logging +logger = logging.getLogger('pymisp') try: from urllib.parse import urljoin + # Least dirty way to support python 2 and 3 + basestring = str + unicode = str except ImportError: from urlparse import urljoin - warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5") + loger.warning("You're using python 2, it is strongly recommended to use python >=3.5") from io import BytesIO, open import zipfile @@ -40,19 +43,6 @@ from .exceptions import PyMISPError, SearchError, MissingDependency, NoURL, NoKe from .mispevent import MISPEvent, MISPAttribute from .abstract import MISPEncode -logger = logging.getLogger(__name__) - - -# Least dirty way to support python 2 and 3 -try: - basestring - unicode - warnings.warn("You're using python 2, it is strongly recommended to use python >=3.4") -except NameError: - basestring = str - unicode = str - - def deprecated(func): '''This is a decorator which can be used to mark functions as deprecated. It will result in a warning being emitted @@ -99,14 +89,14 @@ class PyMISP(object): self.cert = cert self.asynch = asynch if asynch and not ASYNC_OK: - warnings.warn("You turned on Async, but don't have requests_futures installed") + logger.warning("You turned on Async, but don't have requests_futures installed") self.asynch = False self.resources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') if out_type != 'json': raise PyMISPError('The only output type supported by PyMISP is JSON. If you still rely on XML, use PyMISP v2.4.49') if debug is not None: - warnings.warn('debug is deprecated, configure logging in api client') + logger.warning('debug is deprecated, configure logging in your script: import logging; logging.getLogger(\'pymisp\').setLevel(logging.DEBUG)') try: # Make sure the MISP instance is working and the URL is valid @@ -163,6 +153,8 @@ class PyMISP(object): 'Accept': 'application/{}'.format(output), 'content-type': 'application/{}'.format(output), 'User-Agent': 'PyMISP {} - Python {}.{}.{}'.format(__version__, *sys.version_info)}) + if logger.isEnabledFor(logging.DEBUG): + logger.debug(session.headers) return session # ##################### @@ -209,15 +201,16 @@ class PyMISP(object): def _check_response(self, response): """Check if the response from the server is not an unexpected error""" + errors = [] if response.status_code >= 500: - response.raise_for_status() + errors.append(response.json()) + logger.critical('Something bad happened on the server-side: {}'.format(response.json())) try: to_return = response.json() except ValueError: - logger.debug(response.text) - raise PyMISPError('Unknown error: {}'.format(response.text)) + # It the server didn't return a JSON blob, we've a problem. + raise PyMISPError('Unknown error (something is very broken server-side: {}'.format(response.text)) - errors = [] if isinstance(to_return, (list, str)): to_return = {'response': to_return} if to_return.get('error'): @@ -905,8 +898,9 @@ class PyMISP(object): if controller not in ['events', 'attributes']: raise Exception('Invalid controller. Can only be {}'.format(', '.join(['events', 'attributes']))) url = urljoin(self.root_url, '{}/{}'.format(controller, path.lstrip('/'))) - logger.debug('URL: %s', url) - logger.debug('Query: %s', query) + if logger.isEnabledFor(logging.DEBUG): + logger.debug('URL: %s', url) + logger.debug('Query: %s', query) if ASYNC_OK and isinstance(session, FuturesSession) and async_callback: response = session.post(url, data=json.dumps(query), background_callback=async_callback) @@ -1699,6 +1693,8 @@ class PyMISP(object): """Add an object""" session = self.__prepare_session() url = urljoin(self.root_url, 'objects/add/{}/{}'.format(event_id, template_id)) + if logger.isEnabledFor(logging.DEBUG): + logger.debug(url) response = session.post(url, data=misp_object.to_json()) return self._check_response(response) diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index bbf3e45..6b43b68 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit bbf3e45649af5af50c98ad90a86916cf75e8c74d +Subproject commit 6b43b68651a350a26891080ef0feda364b74727a diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 20eeb49..fc435fa 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -16,12 +16,13 @@ from collections import Counter from .abstract import AbstractMISP from .exceptions import UnknownMISPObjectTemplate, InvalidMISPObject, PyMISPError, NewEventError, NewAttributeError +import logging +logger = logging.getLogger('pymisp') import six # Remove that import when discarding python2 support. if six.PY2: - import warnings - warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5") + logger.warning("You're using python 2, it is strongly recommended to use python >=3.5") try: from dateutil.parser import parse diff --git a/pymisp/tools/create_misp_object.py b/pymisp/tools/create_misp_object.py index be16ae3..faa1258 100644 --- a/pymisp/tools/create_misp_object.py +++ b/pymisp/tools/create_misp_object.py @@ -3,7 +3,9 @@ from . import FileObject, PEObject, ELFObject, MachOObject from ..exceptions import MISPObjectException -import warnings +import logging + +logger = logging.getLogger('pymisp') try: import lief @@ -57,15 +59,15 @@ def make_binary_objects(filepath=None, pseudofile=None, filename=None): elif isinstance(lief_parsed, lief.MachO.Binary): return make_macho_objects(lief_parsed, misp_file) except lief.bad_format as e: - warnings.warn('\tBad format: {}'.format(e)) + logger.warning('Bad format: {}'.format(e)) except lief.bad_file as e: - warnings.warn('\tBad file: {}'.format(e)) + logger.warning('Bad file: {}'.format(e)) except lief.parser_error as e: - warnings.warn('\tParser error: {}'.format(e)) + logger.warning('Parser error: {}'.format(e)) except FileTypeNotImplemented as e: # noqa - warnings.warn(e) + logger.warning(e) if not HAS_LIEF: - warnings.warn('Please install lief, documentation here: https://github.com/lief-project/LIEF') + logger.warning('Please install lief, documentation here: https://github.com/lief-project/LIEF') if not filepath: - warnings.warn('LIEF currently requires a filepath and not a pseudo file') + logger.warning('LIEF currently requires a filepath and not a pseudo file') return misp_file, None, None diff --git a/pymisp/tools/elfobject.py b/pymisp/tools/elfobject.py index ee3bd29..d9c9561 100644 --- a/pymisp/tools/elfobject.py +++ b/pymisp/tools/elfobject.py @@ -5,8 +5,9 @@ from .abstractgenerator import AbstractMISPObjectGenerator from ..exceptions import InvalidMISPObject from io import BytesIO from hashlib import md5, sha1, sha256, sha512 -import warnings +import logging +logger = logging.getLogger('pymisp') try: import lief @@ -25,7 +26,7 @@ class ELFObject(AbstractMISPObjectGenerator): def __init__(self, parsed=None, filepath=None, pseudofile=None): if not HAS_PYDEEP: - warnings.warn("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") + logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") if not HAS_LIEF: raise ImportError('Please install lief, documentation here: https://github.com/lief-project/LIEF') if pseudofile: diff --git a/pymisp/tools/fileobject.py b/pymisp/tools/fileobject.py index e75884c..a12d38f 100644 --- a/pymisp/tools/fileobject.py +++ b/pymisp/tools/fileobject.py @@ -8,7 +8,10 @@ from io import BytesIO from hashlib import md5, sha1, sha256, sha512 import math from collections import Counter -import warnings +import logging + +logger = logging.getLogger('pymisp') + try: import pydeep @@ -27,9 +30,9 @@ class FileObject(AbstractMISPObjectGenerator): def __init__(self, filepath=None, pseudofile=None, filename=None): if not HAS_PYDEEP: - warnings.warn("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") + logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") if not HAS_MAGIC: - warnings.warn("Please install python-magic: pip install python-magic.") + logger.warning("Please install python-magic: pip install python-magic.") if filename: # Useful in case the file is copied with a pre-defined name by a script but we want to keep the original name self.__filename = filename diff --git a/pymisp/tools/machoobject.py b/pymisp/tools/machoobject.py index 15663c9..6cf3fa2 100644 --- a/pymisp/tools/machoobject.py +++ b/pymisp/tools/machoobject.py @@ -5,7 +5,9 @@ from ..exceptions import InvalidMISPObject from .abstractgenerator import AbstractMISPObjectGenerator from io import BytesIO from hashlib import md5, sha1, sha256, sha512 -import warnings +import logging + +logger = logging.getLogger('pymisp') try: @@ -25,7 +27,7 @@ class MachOObject(AbstractMISPObjectGenerator): def __init__(self, parsed=None, filepath=None, pseudofile=None): if not HAS_PYDEEP: - warnings.warn("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") + logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") if not HAS_LIEF: raise ImportError('Please install lief, documentation here: https://github.com/lief-project/LIEF') if pseudofile: diff --git a/pymisp/tools/peobject.py b/pymisp/tools/peobject.py index 3467243..2e8bf4a 100644 --- a/pymisp/tools/peobject.py +++ b/pymisp/tools/peobject.py @@ -6,8 +6,9 @@ from .abstractgenerator import AbstractMISPObjectGenerator from io import BytesIO from hashlib import md5, sha1, sha256, sha512 from datetime import datetime -import warnings +import logging +logger = logging.getLogger('pymisp') try: import lief @@ -26,7 +27,7 @@ class PEObject(AbstractMISPObjectGenerator): def __init__(self, parsed=None, filepath=None, pseudofile=None): if not HAS_PYDEEP: - warnings.warn("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") + logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") if not HAS_LIEF: raise ImportError('Please install lief, documentation here: https://github.com/lief-project/LIEF') if pseudofile: From f1a88f460e50ed8eca67b1b04ece7448f4c47705 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 7 Nov 2017 18:19:57 -0800 Subject: [PATCH 013/125] fix: Typo loger -> logger --- pymisp/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/api.py b/pymisp/api.py index 3ec4ed3..a7e993f 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -21,7 +21,7 @@ try: unicode = str except ImportError: from urlparse import urljoin - loger.warning("You're using python 2, it is strongly recommended to use python >=3.5") + logger.warning("You're using python 2, it is strongly recommended to use python >=3.5") from io import BytesIO, open import zipfile From 4512a4eaca040017facb133b78f84e80e265525d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 7 Nov 2017 19:10:54 -0800 Subject: [PATCH 014/125] chg: small improvments in the logging system --- pymisp/__init__.py | 2 +- pymisp/api.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 3edf8b1..ed4316b 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -2,7 +2,7 @@ __version__ = '2.4.81.2' import sys import logging logger = logging.getLogger(__name__) -FORMAT = "[%(filename)s:%(lineno)s - %(funcName)s() ] %(message)s" +FORMAT = "%(levelname)s [%(filename)s:%(lineno)s - %(funcName)s() ] %(message)s" logging.basicConfig(stream=sys.stderr, level=logging.WARNING, format=FORMAT) try: diff --git a/pymisp/api.py b/pymisp/api.py index a7e993f..0908936 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -95,8 +95,9 @@ class PyMISP(object): self.resources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') if out_type != 'json': raise PyMISPError('The only output type supported by PyMISP is JSON. If you still rely on XML, use PyMISP v2.4.49') - if debug is not None: - logger.warning('debug is deprecated, configure logging in your script: import logging; logging.getLogger(\'pymisp\').setLevel(logging.DEBUG)') + if debug: + logger.setLevel(logging.DEBUG) + logger.info('To configure logging in your script, leave it to None and use the following: import logging; logging.getLogger(\'pymisp\').setLevel(logging.DEBUG)') try: # Make sure the MISP instance is working and the URL is valid From f54a029e2a88566ee63011921a32d2d87bccca5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 8 Nov 2017 17:33:55 -0800 Subject: [PATCH 015/125] new: Proper debug system Make it easy to investigate the json blobs sent to the server. --- pymisp/abstract.py | 2 +- pymisp/api.py | 339 +++++++++++++++++-------------------------- pymisp/exceptions.py | 1 + pymisp/mispevent.py | 11 +- 4 files changed, 141 insertions(+), 212 deletions(-) diff --git a/pymisp/abstract.py b/pymisp/abstract.py index 2a5d697..151a3bd 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -64,7 +64,7 @@ class AbstractMISP(collections.MutableMapping): return self.to_dict() def to_json(self): - return json.dumps(self.to_dict(), cls=MISPEncode) + return json.dumps(self, cls=MISPEncode) def __getitem__(self, key): return getattr(self, key) diff --git a/pymisp/api.py b/pymisp/api.py index 0908936..839fe73 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -11,6 +11,14 @@ import base64 import re import functools import logging +import warnings +from io import BytesIO, open +import zipfile + +from . import __version__ +from .exceptions import PyMISPError, SearchError, NoURL, NoKey +from .mispevent import MISPEvent, MISPAttribute +from .abstract import MISPEncode logger = logging.getLogger('pymisp') @@ -22,8 +30,6 @@ try: except ImportError: from urlparse import urljoin logger.warning("You're using python 2, it is strongly recommended to use python >=3.5") -from io import BytesIO, open -import zipfile try: import requests @@ -38,10 +44,6 @@ try: except ImportError: ASYNC_OK = False -from . import __version__ -from .exceptions import PyMISPError, SearchError, MissingDependency, NoURL, NoKey -from .mispevent import MISPEvent, MISPAttribute -from .abstract import MISPEncode def deprecated(func): '''This is a decorator which can be used to mark functions @@ -89,7 +91,7 @@ class PyMISP(object): self.cert = cert self.asynch = asynch if asynch and not ASYNC_OK: - logger.warning("You turned on Async, but don't have requests_futures installed") + logger.critical("You turned on Async, but don't have requests_futures installed") self.asynch = False self.resources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') @@ -118,8 +120,7 @@ class PyMISP(object): raise PyMISPError('Unable to connect to MISP ({}). Please make sure the API key and the URL are correct (http/https is required): {}'.format(self.root_url, e)) try: - session = self.__prepare_session() - response = session.get(urljoin(self.root_url, 'attributes/describeTypes.json')) + response = self.__prepare_request('GET', urljoin(self.root_url, 'attributes/describeTypes.json')) describe_types = self._check_response(response) if describe_types.get('error'): for e in describe_types.get('error'): @@ -137,26 +138,32 @@ class PyMISP(object): self.category_type_mapping = self.describe_types['category_type_mappings'] self.sane_default = self.describe_types['sane_defaults'] - def __prepare_session(self, output='json', async_implemented=False): - """Prepare the session headers""" - - if not HAVE_REQUESTS: - raise MissingDependency('Missing dependency, install requests (`pip install requests`)') - if self.asynch and async_implemented: - session = FuturesSession() + def __prepare_request(self, request_type, url, data=None, + background_callback=None, output_type='json'): + if logger.isEnabledFor(logging.DEBUG): + logger.debug('{} - {}'.format(request_type, url)) + if data is not None: + logger.debug(data) + if data is None: + req = requests.Request(request_type, url) else: - session = requests.Session() - session.verify = self.ssl - session.proxies = self.proxies - session.cert = self.cert - session.headers.update( + req = requests.Request(request_type, url, data=data) + if self.asynch and background_callback is not None: + s = FuturesSession() + else: + s = requests.Session() + prepped = s.prepare_request(req) + prepped.headers.update( {'Authorization': self.key, - 'Accept': 'application/{}'.format(output), - 'content-type': 'application/{}'.format(output), + 'Accept': 'application/{}'.format(output_type), + 'content-type': 'application/{}'.format(output_type), 'User-Agent': 'PyMISP {} - Python {}.{}.{}'.format(__version__, *sys.version_info)}) if logger.isEnabledFor(logging.DEBUG): - logger.debug(session.headers) - return session + logger.debug(prepped.headers) + if self.asynch and background_callback is not None: + return s.send(prepped, verify=self.ssl, proxies=self.proxies, cert=self.cert, background_callback=background_callback) + else: + return s.send(prepped, verify=self.ssl, proxies=self.proxies, cert=self.cert) # ##################### # ### Core helpers #### @@ -294,13 +301,11 @@ class PyMISP(object): Warning, there's a limit on the number of results """ - session = self.__prepare_session() url = urljoin(self.root_url, 'events/index') - if filters is not None: - filters = json.dumps(filters) - response = session.post(url, data=filters) + if filters is None: + response = self.__prepare_request('GET', url) else: - response = session.get(url) + response = self.__prepare_request('POST', url, json.dumps(filters)) return self._check_response(response) def get_event(self, event_id): @@ -308,9 +313,8 @@ class PyMISP(object): :param event_id: Event id to get """ - session = self.__prepare_session() url = urljoin(self.root_url, 'events/{}'.format(event_id)) - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def add_event(self, event): @@ -318,14 +322,12 @@ class PyMISP(object): :param event: Event as JSON object / string to add """ - session = self.__prepare_session() url = urljoin(self.root_url, 'events') if isinstance(event, MISPEvent): - event = json.dumps(event, cls=MISPEncode) - if isinstance(event, basestring): - response = session.post(url, data=event) - else: - response = session.post(url, data=json.dumps(event)) + event = event.to_json() + elif not isinstance(event, basestring): + event = json.dumps(event) + response = self.__prepare_request('POST', url, event) return self._check_response(response) def update_event(self, event_id, event): @@ -334,14 +336,12 @@ class PyMISP(object): :param event_id: Event id to update :param event: Event as JSON object / string to add """ - session = self.__prepare_session() url = urljoin(self.root_url, 'events/{}'.format(event_id)) if isinstance(event, MISPEvent): - event = json.dumps(event, cls=MISPEncode) - if isinstance(event, basestring): - response = session.post(url, data=event) - else: - response = session.post(url, data=json.dumps(event)) + event = event.to_json() + elif not isinstance(event, basestring): + event = json.dumps(event) + response = self.__prepare_request('POST', url, event) return self._check_response(response) def delete_event(self, event_id): @@ -349,26 +349,23 @@ class PyMISP(object): :param event_id: Event id to delete """ - session = self.__prepare_session() url = urljoin(self.root_url, 'events/{}'.format(event_id)) - response = session.delete(url) + response = self.__prepare_request('DELETE', url) return self._check_response(response) def delete_attribute(self, attribute_id, hard_delete=False): """Delete an attribute by ID""" - session = self.__prepare_session() if hard_delete: url = urljoin(self.root_url, 'attributes/delete/{}/1'.format(attribute_id)) else: url = urljoin(self.root_url, 'attributes/delete/{}'.format(attribute_id)) - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def pushEventToZMQ(self, event_id): """Force push an event on ZMQ""" - session = self.__prepare_session() url = urljoin(self.root_url, 'events/pushEventToZMQ/{}.json'.format(event_id)) - response = session.post(url) + response = self.__prepare_request('POST', url) return self._check_response(response) # ############################################## @@ -401,12 +398,11 @@ class PyMISP(object): event_id = full_event.id if full_event.published: return {'error': 'Already published'} - session = self.__prepare_session() if not alert: url = urljoin(self.root_url, 'events/publish/{}'.format(event_id)) else: url = urljoin(self.root_url, 'events/alert/{}'.format(event_id)) - response = session.post(url) + response = self.__prepare_request('POST', url) return self._check_response(response) def change_threat_level(self, event, threat_level_id): @@ -431,20 +427,18 @@ class PyMISP(object): """Tag an event or an attribute""" if not self._valid_uuid(uuid): raise PyMISPError('Invalid UUID') - session = self.__prepare_session() + url = urljoin(self.root_url, 'tags/attachTagToObject') to_post = {'uuid': uuid, 'tag': tag} - path = 'tags/attachTagToObject' - response = session.post(urljoin(self.root_url, path), data=json.dumps(to_post)) + response = self.__prepare_request('POST', url, json.dumps(to_post)) return self._check_response(response) def untag(self, uuid, tag): """Untag an event or an attribute""" if not self._valid_uuid(uuid): raise PyMISPError('Invalid UUID') - session = self.__prepare_session() + url = urljoin(self.root_url, 'tags/removeTagFromObject') to_post = {'uuid': uuid, 'tag': tag} - path = 'tags/removeTagFromObject' - response = session.post(urljoin(self.root_url, path), data=json.dumps(to_post)) + response = self.__prepare_request('POST', url, json.dumps(to_post)) return self._check_response(response) # ##### File attributes ##### @@ -474,9 +468,8 @@ class PyMISP(object): if proposal: response = self.proposal_add(eventID_to_update, a) else: - session = self.__prepare_session() url = urljoin(self.root_url, 'attributes/add/{}'.format(eventID_to_update)) - response = self._check_response(session.post(url, data=json.dumps(a, cls=MISPEncode))) + response = self.__prepare_request('POST', url, a.to_json()) return response def add_named_attribute(self, event, type_value, value, category=None, to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): @@ -803,61 +796,54 @@ class PyMISP(object): def _upload_sample(self, to_post, event_id=None): """Helper to upload a sample""" - session = self.__prepare_session() if event_id is None: url = urljoin(self.root_url, 'events/upload_sample') else: url = urljoin(self.root_url, 'events/upload_sample/{}'.format(event_id)) - logger.info(to_post) - response = session.post(url, data=json.dumps(to_post)) + response = self.__prepare_request('POST', url, json.dumps(to_post)) return self._check_response(response) # ############################ # ######## Proposals ######### # ############################ - def __query_proposal(self, session, path, id, attribute=None): + def __query_proposal(self, path, id, attribute=None): """Helper to prepare a query to handle proposals""" url = urljoin(self.root_url, 'shadow_attributes/{}/{}'.format(path, id)) if path in ['add', 'edit']: query = {'request': {'ShadowAttribute': attribute}} - response = session.post(url, data=json.dumps(query, cls=MISPEncode)) + response = self.__prepare_request('POST', url, json.dumps(query, cls=MISPEncode)) elif path == 'view': - response = session.get(url) + response = self.__prepare_request('GET', url) else: # accept or discard - response = session.post(url) + response = self.__prepare_request('POST', url) return self._check_response(response) def proposal_view(self, event_id=None, proposal_id=None): """View a proposal""" - session = self.__prepare_session() if proposal_id is not None and event_id is not None: return {'error': 'You can only view an event ID or a proposal ID'} if event_id is not None: id = event_id else: id = proposal_id - return self.__query_proposal(session, 'view', id) + return self.__query_proposal('view', id) def proposal_add(self, event_id, attribute): """Add a proposal""" - session = self.__prepare_session() - return self.__query_proposal(session, 'add', event_id, attribute) + return self.__query_proposal('add', event_id, attribute) def proposal_edit(self, attribute_id, attribute): """Edit a proposal""" - session = self.__prepare_session() - return self.__query_proposal(session, 'edit', attribute_id, attribute) + return self.__query_proposal('edit', attribute_id, attribute) def proposal_accept(self, proposal_id): """Accept a proposal""" - session = self.__prepare_session() - return self.__query_proposal(session, 'accept', proposal_id) + return self.__query_proposal('accept', proposal_id) def proposal_discard(self, proposal_id): """Discard a proposal""" - session = self.__prepare_session() - return self.__query_proposal(session, 'discard', proposal_id) + return self.__query_proposal('discard', proposal_id) # ############################## # ###### Attribute update ###### @@ -868,8 +854,7 @@ class PyMISP(object): if to_ids not in [0, 1]: raise Exception('to_ids can only be 0 or 1') query = {"to_ids": to_ids} - session = self.__prepare_session() - return self.__query(session, 'edit/{}'.format(attribute_uuid), query, controller='attributes') + return self.__query('edit/{}'.format(attribute_uuid), query, controller='attributes') # ############################## # ###### Attribute update ###### @@ -885,28 +870,24 @@ class PyMISP(object): query['adhereToWarninglists'] = adhereToWarninglists if distribution is not None: query['distribution'] = distribution - session = self.__prepare_session() - return self.__query(session, 'freeTextImport/{}'.format(event_id), query, controller='events') + return self.__query('freeTextImport/{}'.format(event_id), query, controller='events') # ############################## # ######## REST Search ######### # ############################## - def __query(self, session, path, query, controller='events', async_callback=None): + def __query(self, path, query, controller='events', async_callback=None): """Helper to prepare a search query""" if query.get('error') is not None: return query if controller not in ['events', 'attributes']: raise Exception('Invalid controller. Can only be {}'.format(', '.join(['events', 'attributes']))) url = urljoin(self.root_url, '{}/{}'.format(controller, path.lstrip('/'))) - if logger.isEnabledFor(logging.DEBUG): - logger.debug('URL: %s', url) - logger.debug('Query: %s', query) - if ASYNC_OK and isinstance(session, FuturesSession) and async_callback: - response = session.post(url, data=json.dumps(query), background_callback=async_callback) + if ASYNC_OK and async_callback: + response = self.__prepare_request('POST', url, json.dumps(query), async_callback) else: - response = session.post(url, data=json.dumps(query)) + response = self.__prepare_request('POST', url, json.dumps(query)) return self._check_response(response) def search_index(self, published=None, eventid=None, tag=None, datefrom=None, @@ -952,13 +933,12 @@ class PyMISP(object): if not set(param).issubset(rule_levels[rule]): raise SearchError('Values in your {} are invalid, has to be in {}'.format(rule, ', '.join(str(x) for x in rule_levels[rule]))) to_post[rule] = '|'.join(str(x) for x in param) - session = self.__prepare_session(async_implemented=(async_callback is not None)) url = urljoin(self.root_url, buildup_url) if self.asynch and async_callback: - response = session.post(url, data=json.dumps(to_post), background_callback=async_callback) + response = self.__prepare_request('POST', url, json.dumps(to_post), async_callback) else: - response = session.post(url, data=json.dumps(to_post)) + response = self.__prepare_request('POST', url, json.dumps(to_post)) res = self._check_response(response) if normalize: to_return = {'response': []} @@ -971,8 +951,7 @@ class PyMISP(object): def search_all(self, value): """Search a value in the whole database""" query = {'value': value, 'searchall': 1} - session = self.__prepare_session() - return self.__query(session, 'restSearch/download', query) + return self.__query('restSearch/download', query) def __prepare_rest_search(self, values, not_values): """Prepare a search, generate the chain processed by the server @@ -1082,8 +1061,7 @@ class PyMISP(object): raise SearchError('Unused parameter: {}'.format(', '.join(kwargs.keys()))) # Create a session, make it async if and only if we have a callback - session = self.__prepare_session(async_implemented=(async_callback is not None)) - return self.__query(session, 'restSearch/download', query, controller, async_callback) + return self.__query('restSearch/download', query, controller, async_callback) def get_attachment(self, attribute_id): """Get an attachement (not a malware sample) by attribute ID. @@ -1091,9 +1069,8 @@ class PyMISP(object): :param attribute_id: Attribute ID to fetched """ - attach = urljoin(self.root_url, 'attributes/downloadAttachment/download/{}'.format(attribute_id)) - session = self.__prepare_session() - response = session.get(attach) + url = urljoin(self.root_url, 'attributes/downloadAttachment/download/{}'.format(attribute_id)) + response = self.__prepare_request('GET', url) try: response.json() # The query fails, response contains a json blob @@ -1104,9 +1081,9 @@ class PyMISP(object): def get_yara(self, event_id): """Get the yara rules from an event""" + url = urljoin(self.root_url, 'attributes/restSearch') to_post = {'request': {'eventid': event_id, 'type': 'yara'}} - session = self.__prepare_session() - response = session.post(urljoin(self.root_url, 'attributes/restSearch'), data=json.dumps(to_post)) + response = self.__prepare_request('POST', url, data=json.dumps(to_post)) result = self._check_response(response) if result.get('error') is not None: return False, result.get('error') @@ -1117,9 +1094,9 @@ class PyMISP(object): def download_samples(self, sample_hash=None, event_id=None, all_samples=False): """Download samples, by hash or event ID. If there are multiple samples in one event, use the all_samples switch""" + url = urljoin(self.root_url, 'attributes/downloadSample') to_post = {'request': {'hash': sample_hash, 'eventID': event_id, 'allSamples': all_samples}} - session = self.__prepare_session() - response = session.post(urljoin(self.root_url, 'attributes/downloadSample'), data=json.dumps(to_post)) + response = self.__prepare_request('POST', url, data=json.dumps(to_post)) result = self._check_response(response) if result.get('error') is not None: return False, result.get('error') @@ -1155,9 +1132,8 @@ class PyMISP(object): def get_all_tags(self, quiet=False): """Get all the tags used on the instance""" - session = self.__prepare_session() url = urljoin(self.root_url, 'tags') - response = session.get(url) + response = self.__prepare_request('GET', url) r = self._check_response(response) if not quiet or r.get('errors'): return r @@ -1170,9 +1146,8 @@ class PyMISP(object): def new_tag(self, name=None, colour="#00ace6", exportable=False): """Create a new tag""" to_post = {'Tag': {'name': name, 'colour': colour, 'exportable': exportable}} - session = self.__prepare_session() url = urljoin(self.root_url, 'tags/add') - response = session.post(url, data=json.dumps(to_post)) + response = self.__prepare_request('POST', url, json.dumps(to_post)) return self._check_response(response) # ########## Version ########## @@ -1192,16 +1167,14 @@ class PyMISP(object): def get_recommended_api_version(self): """Returns the recommended API version from the server""" - session = self.__prepare_session() url = urljoin(self.root_url, 'servers/getPyMISPVersion.json') - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def get_version(self): """Returns the version of the instance.""" - session = self.__prepare_session() url = urljoin(self.root_url, 'servers/getVersion.json') - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def get_version_master(self): @@ -1217,19 +1190,17 @@ class PyMISP(object): def get_attributes_statistics(self, context='type', percentage=None): """Get attributes statistics from the MISP instance""" - session = self.__prepare_session() if (context != 'category'): context = 'type' if percentage is not None: url = urljoin(self.root_url, 'attributes/attributeStatistics/{}/{}'.format(context, percentage)) else: url = urljoin(self.root_url, 'attributes/attributeStatistics/{}'.format(context)) - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def get_tags_statistics(self, percentage=None, name_sort=None): """Get tags statistics from the MISP instance""" - session = self.__prepare_session() if percentage is not None: percentage = 'true' else: @@ -1239,32 +1210,29 @@ class PyMISP(object): else: name_sort = 'false' url = urljoin(self.root_url, 'tags/tagStatistics/{}/{}'.format(percentage, name_sort)) - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) # ############## Sightings ################## def sighting_per_id(self, attribute_id): """Add a sighting to an attribute (by attribute ID)""" - session = self.__prepare_session() url = urljoin(self.root_url, 'sightings/add/{}'.format(attribute_id)) - response = session.post(url) + response = self.__prepare_request('POST', url) return self._check_response(response) def sighting_per_uuid(self, attribute_uuid): """Add a sighting to an attribute (by attribute UUID)""" - session = self.__prepare_session() url = urljoin(self.root_url, 'sightings/add/{}'.format(attribute_uuid)) - response = session.post(url) + response = self.__prepare_request('POST', url) return self._check_response(response) def set_sightings(self, sightings): """Push a sighting (python dictionary)""" if isinstance(sightings, dict): sightings = json.dumps(sightings) - session = self.__prepare_session() url = urljoin(self.root_url, 'sightings/add/') - response = session.post(url, data=sightings) + response = self.__prepare_request('POST', url, sightings) return self._check_response(response) def sighting_per_json(self, json_file): @@ -1277,9 +1245,8 @@ class PyMISP(object): def get_sharing_groups(self): """Get the existing sharing groups""" - session = self.__prepare_session() url = urljoin(self.root_url, 'sharing_groups.json') - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response)['response'] # ############## Users ################## @@ -1325,57 +1292,49 @@ class PyMISP(object): return user def get_users_list(self): - session = self.__prepare_session() url = urljoin(self.root_url, 'admin/users') - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response)['response'] def get_user(self, user_id): - session = self.__prepare_session() url = urljoin(self.root_url, 'admin/users/view/{}'.format(user_id)) - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def add_user(self, email, org_id, role_id, **kwargs): - new_user = self._set_user_parameters(**dict(email=email, org_id=org_id, role_id=role_id, **kwargs)) - session = self.__prepare_session() url = urljoin(self.root_url, 'admin/users/add/') - response = session.post(url, data=json.dumps(new_user)) + new_user = self._set_user_parameters(**dict(email=email, org_id=org_id, role_id=role_id, **kwargs)) + response = self.__prepare_request('POST', url, json.dumps(new_user)) return self._check_response(response) def add_user_json(self, json_file): - session = self.__prepare_session() with open(json_file, 'r') as f: jdata = json.load(f) url = urljoin(self.root_url, 'admin/users/add/') - response = session.post(url, data=json.dumps(jdata)) + response = self.__prepare_request('POST', url, json.dumps(jdata)) return self._check_response(response) def get_user_fields_list(self): - session = self.__prepare_session() url = urljoin(self.root_url, 'admin/users/add/') - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def edit_user(self, user_id, **kwargs): edit_user = self._set_user_parameters(**kwargs) - session = self.__prepare_session() url = urljoin(self.root_url, 'admin/users/edit/{}'.format(user_id)) - response = session.post(url, data=json.dumps(edit_user)) + response = self.__prepare_request('POST', url, json.dumps(edit_user)) return self._check_response(response) def edit_user_json(self, json_file, user_id): - session = self.__prepare_session() with open(json_file, 'r') as f: jdata = json.load(f) url = urljoin(self.root_url, 'admin/users/edit/{}'.format(user_id)) - response = session.post(url, data=json.dumps(jdata)) + response = self.__prepare_request('POST', url, json.dumps(jdata)) return self._check_response(response) def delete_user(self, user_id): - session = self.__prepare_session() url = urljoin(self.root_url, 'admin/users/delete/{}'.format(user_id)) - response = session.post(url) + response = self.__prepare_request('POST', url) return self._check_response(response) # ############## Organisations ################## @@ -1401,64 +1360,56 @@ class PyMISP(object): return organisation def get_organisations_list(self, scope="local"): - session = self.__prepare_session() scope = scope.lower() if scope not in ["local", "external", "all"]: raise ValueError("Authorized fields are 'local','external' or 'all'") url = urljoin(self.root_url, 'organisations/index/scope:{}'.format(scope)) - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response)['response'] def get_organisation(self, organisation_id): - session = self.__prepare_session() url = urljoin(self.root_url, 'organisations/view/{}'.format(organisation_id)) - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def add_organisation(self, name, **kwargs): new_org = self._set_organisation_parameters(**dict(name=name, **kwargs)) - session = self.__prepare_session() if 'local' in new_org: if new_org.get('local') is False: if 'uuid' not in new_org: raise PyMISPError('A remote org MUST have a valid uuid') url = urljoin(self.root_url, 'admin/organisations/add/') - response = session.post(url, data=json.dumps(new_org)) + response = self.__prepare_request('POST', url, json.dumps(new_org)) return self._check_response(response) def add_organisation_json(self, json_file): - session = self.__prepare_session() with open(json_file, 'r') as f: jdata = json.load(f) url = urljoin(self.root_url, 'admin/organisations/add/') - response = session.post(url, data=json.dumps(jdata)) + response = self.__prepare_request('POST', url, json.dumps(jdata)) return self._check_response(response) def get_organisation_fields_list(self): - session = self.__prepare_session() url = urljoin(self.root_url, 'admin/organisations/add/') - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def edit_organisation(self, org_id, **kwargs): edit_org = self._set_organisation_parameters(**kwargs) - session = self.__prepare_session() url = urljoin(self.root_url, 'admin/organisations/edit/{}'.format(org_id)) - response = session.post(url, data=json.dumps(edit_org)) + response = self.__prepare_request('POST', url, json.dumps(edit_org)) return self._check_response(response) def edit_organisation_json(self, json_file, org_id): - session = self.__prepare_session() with open(json_file, 'r') as f: jdata = json.load(f) url = urljoin(self.root_url, 'admin/organisations/edit/{}'.format(org_id)) - response = session.post(url, data=json.dumps(jdata)) + response = self.__prepare_request('POST', url, json.dumps(jdata)) return self._check_response(response) def delete_organisation(self, org_id): - session = self.__prepare_session() url = urljoin(self.root_url, 'admin/organisations/delete/{}'.format(org_id)) - response = session.post(url) + response = self.__prepare_request('POST', url) return self._check_response(response) # ############## Servers ################## @@ -1521,17 +1472,15 @@ class PyMISP(object): new_server = self._set_server_parameters(url, name, authkey, organisation, internal, push, pull, self_signed, push_rules, pull_rules, submitted_cert, submitted_client_cert, None, None) - session = self.__prepare_session() url = urljoin(self.root_url, 'servers/add') - response = session.post(url, data=json.dumps(new_server)) + response = self.__prepare_request('POST', url, json.dumps(new_server)) return self._check_response(response) def add_server_json(self, json_file): - session = self.__prepare_session() with open(json_file, 'r') as f: jdata = json.load(f) url = urljoin(self.root_url, 'servers/add') - response = session.post(url, data=json.dumps(jdata)) + response = self.__prepare_request('POST', url, json.dumps(jdata)) return self._check_response(response) def edit_server(self, server_id, url=None, name=None, authkey=None, organisation=None, internal=None, push=False, @@ -1540,35 +1489,31 @@ class PyMISP(object): new_server = self._set_server_parameters(url, name, authkey, organisation, internal, push, pull, self_signed, push_rules, pull_rules, submitted_cert, submitted_client_cert, delete_cert, delete_client_cert) - session = self.__prepare_session() url = urljoin(self.root_url, 'servers/edit/{}'.format(server_id)) - response = session.post(url, data=json.dumps(new_server)) + response = self.__prepare_request('POST', url, json.dumps(new_server)) return self._check_response(response) def edit_server_json(self, json_file, server_id): - session = self.__prepare_session() with open(json_file, 'r') as f: jdata = json.load(f) url = urljoin(self.root_url, 'servers/edit/{}'.format(server_id)) - response = session.post(url, data=json.dumps(jdata)) + response = self.__prepare_request('POST', url, json.dumps(jdata)) return self._check_response(response) # ############## Roles ################## def get_roles_list(self): """Get the list of existing roles""" - session = self.__prepare_session() url = urljoin(self.root_url, '/roles') - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response)['response'] # ############## Tags ################## def get_tags_list(self): """Get the list of existing tags""" - session = self.__prepare_session() url = urljoin(self.root_url, '/tags') - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response)['Tag'] # ############################################## @@ -1579,9 +1524,8 @@ class PyMISP(object): def download_all_suricata(self): """Download all suricata rules events.""" - suricata_rules = urljoin(self.root_url, 'events/nids/suricata/download') - session = self.__prepare_session('rules') - response = session.get(suricata_rules) + url = urljoin(self.root_url, 'events/nids/suricata/download') + response = self.__prepare_request('GET', url, output_type='rules') return response def download_suricata_rule_event(self, event_id): @@ -1589,18 +1533,16 @@ class PyMISP(object): :param event_id: ID of the event to download (same as get) """ - template = urljoin(self.root_url, 'events/nids/suricata/download/{}'.format(event_id)) - session = self.__prepare_session('rules') - response = session.get(template) + url = urljoin(self.root_url, 'events/nids/suricata/download/{}'.format(event_id)) + response = self.__prepare_request('GET', url, output_type='rules') return response # ############## Text ############### def get_all_attributes_txt(self, type_attr, tags=False, eventId=False, allowNonIDS=False, date_from=False, date_to=False, last=False, enforceWarninglist=False, allowNotPublished=False): """Get all attributes from a specific type as plain text. Only published and IDS flagged attributes are exported, except if stated otherwise.""" - session = self.__prepare_session('txt') url = urljoin(self.root_url, 'attributes/text/download/%s/%s/%s/%s/%s/%s/%s/%s/%s' % (type_attr, tags, eventId, allowNonIDS, date_from, date_to, last, enforceWarninglist, allowNotPublished)) - response = session.get(url) + response = self.__prepare_request('GET', url, output_type='txt') return response # ############## STIX ############## @@ -1610,12 +1552,10 @@ class PyMISP(object): if tags: if isinstance(tags, list): tags = "&&".join(tags) - - session = self.__prepare_session() url = urljoin(self.root_url, "/events/stix/download/{}/{}/{}/{}/{}".format( event_id, with_attachments, tags, from_date, to_date)) logger.debug("Getting STIX event from %s", url) - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def get_stix(self, **kwargs): @@ -1627,58 +1567,50 @@ class PyMISP(object): def fetch_feed(self, feed_id): """Fetch one single feed""" - session = self.__prepare_session() url = urljoin(self.root_url, 'feeds/fetchFromFeed/{}'.format(feed_id)) - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def view_feeds(self): """Get the content of all the feeds""" - session = self.__prepare_session() url = urljoin(self.root_url, 'feeds') - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def view_feed(self, feed_ids): """Get the content of a single feed""" - session = self.__prepare_session() url = urljoin(self.root_url, 'feeds/view/{}'.format(feed_ids)) - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def cache_feeds_all(self): """ Cache all the feeds""" - session = self.__prepare_session() url = urljoin(self.root_url, 'feeds/cacheFeeds/all') - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def cache_feed(self, feed_id): """Cache a specific feed""" - session = self.__prepare_session() url = urljoin(self.root_url, 'feeds/cacheFeeds/{}'.format(feed_id)) - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def cache_feeds_freetext(self): """Cache all the freetext feeds""" - session = self.__prepare_session() url = urljoin(self.root_url, 'feeds/cacheFeeds/freetext') - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def cache_feeds_misp(self): """Cache all the MISP feeds""" - session = self.__prepare_session() url = urljoin(self.root_url, 'feeds/cacheFeeds/misp') - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def compare_feeds(self): """Generate the comparison matrix for all the MISP feeds""" - session = self.__prepare_session() url = urljoin(self.root_url, 'feeds/compareFeeds') - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def cache_all_feeds(self): @@ -1692,25 +1624,20 @@ class PyMISP(object): def add_object(self, event_id, template_id, misp_object): """Add an object""" - session = self.__prepare_session() url = urljoin(self.root_url, 'objects/add/{}/{}'.format(event_id, template_id)) - if logger.isEnabledFor(logging.DEBUG): - logger.debug(url) - response = session.post(url, data=misp_object.to_json()) + response = self.__prepare_request('POST', url, misp_object.to_json()) return self._check_response(response) def add_object_reference(self, misp_object_reference): """Add a reference to an object""" - session = self.__prepare_session() url = urljoin(self.root_url, 'object_references/add') - response = session.post(url, data=misp_object_reference.to_json()) + response = self.__prepare_request('POST', url, misp_object_reference.to_json()) return self._check_response(response) def get_object_templates_list(self): """Returns the list of Object templates available on the MISP instance""" - session = self.__prepare_session() url = urljoin(self.root_url, 'objectTemplates') - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response)['response'] def get_object_template_id(self, object_uuid): @@ -1727,7 +1654,6 @@ class PyMISP(object): @deprecated def add_tag(self, event, tag, attribute=False): - session = self.__prepare_session() if attribute: to_post = {'request': {'Attribute': {'id': event['id'], 'tag': tag}}} path = 'attributes/addTag' @@ -1737,17 +1663,18 @@ class PyMISP(object): event = event["Event"] to_post = {'request': {'Event': {'id': event['id'], 'tag': tag}}} path = 'events/addTag' - response = session.post(urljoin(self.root_url, path), data=json.dumps(to_post)) + url = urljoin(self.root_url, path) + response = self.__prepare_request('POST', url, json.dumps(to_post)) return self._check_response(response) @deprecated def remove_tag(self, event, tag, attribute=False): - session = self.__prepare_session() if attribute: to_post = {'request': {'Attribute': {'id': event['id'], 'tag': tag}}} path = 'attributes/removeTag' else: to_post = {'request': {'Event': {'id': event['Event']['id'], 'tag': tag}}} path = 'events/removeTag' - response = session.post(urljoin(self.root_url, path), data=json.dumps(to_post)) + url = urljoin(self.root_url, path) + response = self.__prepare_request('POST', url, json.dumps(to_post)) return self._check_response(response) diff --git a/pymisp/exceptions.py b/pymisp/exceptions.py index 8a09c10..d828e74 100644 --- a/pymisp/exceptions.py +++ b/pymisp/exceptions.py @@ -40,6 +40,7 @@ class InvalidMISPObject(MISPObjectException): """Exception raised when an object doesn't respect the contrains in the definition""" pass + class UnknownMISPObjectTemplate(MISPObjectException): """Exception raised when the template is unknown""" pass diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index fc435fa..645dc9b 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -16,10 +16,11 @@ from collections import Counter from .abstract import AbstractMISP from .exceptions import UnknownMISPObjectTemplate, InvalidMISPObject, PyMISPError, NewEventError, NewAttributeError +import six # Remove that import when discarding python2 support. + import logging logger = logging.getLogger('pymisp') -import six # Remove that import when discarding python2 support. if six.PY2: logger.warning("You're using python 2, it is strongly recommended to use python >=3.5") @@ -139,7 +140,7 @@ class MISPAttribute(AbstractMISP): try: c.verify(signed_data, signature=base64.b64decode(self.sig), verify=keys[:1]) return {self.uuid: True} - except: + except Exception: return {self.uuid: False} def set_all_values(self, **kwargs): @@ -252,7 +253,7 @@ class MISPAttribute(AbstractMISP): else: with f.open(name, pwd=b'infected') as unpacked: self._malware_binary = BytesIO(unpacked.read()) - except: + except Exception: # not a encrypted zip file, assuming it is a new malware sample self._prepare_new_malware_sample() @@ -383,7 +384,7 @@ class MISPEvent(AbstractMISP): try: c.verify(signed_data, signature=base64.b64decode(self.sig), verify=keys[:1]) to_return[self.uuid] = True - except: + except Exception: to_return[self.uuid] = False for a in self.attributes: to_return.update(a.verify(gpg_uid)) @@ -393,7 +394,7 @@ class MISPEvent(AbstractMISP): try: c.verify(to_verify_global, signature=base64.b64decode(self.global_sig), verify=keys[:1]) to_return['global'] = True - except: + except Exception: to_return['global'] = False return to_return From e6ab90012d5c5e1fd1c0973360ac87f18c0a742c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 8 Nov 2017 18:01:27 -0800 Subject: [PATCH 016/125] chg: Update readme for new logging system --- README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/README.md b/README.md index b8a77c0..de40a54 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,23 @@ cd examples python3 last.py -l 10 ``` +## Debugging + +You have two options there: + +1. Pass `debug=True` to `PyMISP` and it will enable logging.DEBUG to stderr on the whole module + +2. Use the python logging module directly: + +```python + +import logging +logger = logging.getLogger('pymisp') + +# Configure it as you whish, for example, enable DEBUG mode: +logger.setLevel(logging.DEBUG) +``` + ## Documentation [PyMISP API documentation is available](https://media.readthedocs.org/pdf/pymisp/master/pymisp.pdf). From 5d414cb8e95070fdb0b09b1fb39f79319e035150 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 9 Nov 2017 13:45:31 -0800 Subject: [PATCH 017/125] chg: Bump misp-objects --- pymisp/data/misp-objects | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index 6b43b68..66c4578 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit 6b43b68651a350a26891080ef0feda364b74727a +Subproject commit 66c4578f08efc6b92737f1667cbaf237149a8e46 From c18c4538a5e3b3017fe2d1fe5e7e5e5b8621cf81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 9 Nov 2017 13:47:17 -0800 Subject: [PATCH 018/125] chg: Bump CHANGELOG --- CHANGELOG.txt | 72 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index f6e5ebf..8fc6177 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -2,6 +2,78 @@ Changelog ========= +v2.4.82 (2017-11-09) +-------------------- + +New +~~~ +- Proper debug system. [Raphaël Vinot] + + Make it easy to investigate the json blobs sent to the server. + +Changes +~~~~~~~ +- Bump misp-objects. [Raphaël Vinot] +- Update readme for new logging system. [Raphaël Vinot] +- Small improvments in the logging system. [Raphaël Vinot] +- Properly use python logging module. [Raphaël Vinot] +- Update asciidoctor generator. [Raphaël Vinot] +- Remove warning if PyMISP is too new. [Raphaël Vinot] +- Add simple asciidoc generator for MISP event. [Raphaël Vinot] +- Update changelog. [Raphaël Vinot] + +Fix +~~~ +- Typo loger -> logger. [Raphaël Vinot] +- Let load unknown object relations in known templates. [Raphaël Vinot] + + This isn't recommended, but happens very often. +- Allow to load non-malware ZIP files in MISP Event. [Raphaël Vinot] + + Prior to his patch, any zip file loaded by MISP Event was unpacked and + processed as an excrypted malware from MISP. +- Properly pass the distribution when uploading a sample. [Raphaël + Vinot] +- Properly upload a sample in an existing event. [Raphaël Vinot] + + Fix https://github.com/MISP/PyMISP/issues/123 +- Properly set the distribution at event level. [Raphaël Vinot] + + fix #120 +- Properly pop the distribution key. [Raphaël Vinot] +- Update dependencies for VT generator. [Raphaël Vinot] + +Other +~~~~~ +- Merge pull request #126 from CenturyLinkCIRT/master. [Raphaël Vinot] + + Added vt_to_misp.py example and VTReportObject +- Merge branch 'master' of https://github.com/MISP/PyMISP. [Thomas + Gardner] +- Fix test suite. [Raphaël Vinot] +- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot] +- Merge pull request #122 from LDO-CERT/master. [Raphaël Vinot] + + Created add_generic_object.py +- Created add_generic_object.py. [garanews] + + usage: add_generic_object.py [-h] -e EVENT -t TYPE -d DICT + + Examples: + python3 add_generic_object.py -e 1683 -t email -d '{"subject":"The Pink Letter", "to":"jon@snow.org"}' + python3 add_generic_object.py -e 2343 -t person -d '{"first-name":"Daenerys", "last-name":"Targaryen", "place-of-birth":"Dragonstone"}' + python3 add_generic_object.py -e 3596 -t "domain|ip" -d '{"domain":"stormborn.org", "ip":"50.63.202.33"}' +- Added vtreportobject and vt_to_misp example. [Thomas Gardner] +- Created add_generic_object.py. [garanews] + + usage: add_generic_object.py [-h] -e EVENT -t TYPE -d DICT + + Examples: + python3 add_generic_object.py -e 1683 -t email -d '{"subject":"The Pink Letter", "to":"jon@snow.org"}' + python3 add_generic_object.py -e 2343 -t person -d '{"first-name":"Daenerys", "last-name":"Targaryen", "place-of-birth":"Dragonstone"}' + python3 add_generic_object.py -e 3596 -t "domain|ip" -d '{"domain":"stormborn.org", "ip":"50.63.202.33"}' + + v2.4.81.2 (2017-10-24) ---------------------- From 31e98c6e5d7b77f2b94df0c9b601ed1e57422519 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 9 Nov 2017 13:49:09 -0800 Subject: [PATCH 019/125] chg: Bump PyMISP version --- pymisp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index ed4316b..443e9dc 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.4.81.2' +__version__ = '2.4.82' import sys import logging logger = logging.getLogger(__name__) From 55b4064a57b726baed4bf80687a4c358fb39fb7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 10 Nov 2017 14:56:53 -0800 Subject: [PATCH 020/125] chg: Improve documentation Fix #121 --- README.md | 10 ++++++++++ pymisp/api.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index de40a54..bf44f06 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,16 @@ logger = logging.getLogger('pymisp') logger.setLevel(logging.DEBUG) ``` +Or if you want to write the debug output to a file instead of stderr: + +```python +import pymisp +import logging + +logger = logging.getLogger('pymisp') +logging.basicConfig(level=logging.DEBUG, filename="debug.log", filemode='w', format=pymisp.FORMAT) +``` + ## Documentation [PyMISP API documentation is available](https://media.readthedocs.org/pdf/pymisp/master/pymisp.pdf). diff --git a/pymisp/api.py b/pymisp/api.py index 839fe73..78901ab 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -72,7 +72,7 @@ class PyMISP(object): signed certiifcate (the concatenation of all the *.crt of the chain) :param out_type: Type of object (json) NOTE: XML output isn't supported anymore, keeping the flag for compatibility reasons. - :param debug: deprecated, configure logging in api client instead + :param debug: Write all the debug information to stderr :param proxies: Proxy dict as describes here: http://docs.python-requests.org/en/master/user/advanced/#proxies :param cert: Client certificate, as described there: http://docs.python-requests.org/en/master/user/advanced/#ssl-cert-verification :param asynch: Use asynchronous processing where possible From b1262a0c961f5ab64bc2d90f3df0d1018a2d4045 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 10 Nov 2017 15:42:07 -0800 Subject: [PATCH 021/125] chg: Add fast publish method Fix #86 --- pymisp/api.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 78901ab..25c6bb4 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -385,6 +385,18 @@ class PyMISP(object): eid = e.id return self.update_event(eid, e) + def fast_publish(self, event_id, alert=False): + """Does the same as the publish method, but just try to publish the event + even with one single HTTP GET. + The default is to not send a mail as it is assumed this method is called on update. + """ + if not alert: + url = urljoin(self.root_url, 'events/publish/{}'.format(event_id)) + else: + url = urljoin(self.root_url, 'events/alert/{}'.format(event_id)) + response = self.__prepare_request('POST', url) + return self._check_response(response) + def publish(self, event, alert=True): """Publish event (with or without alert email) :param event: pass event or event id (as string or int) to publish @@ -398,12 +410,7 @@ class PyMISP(object): event_id = full_event.id if full_event.published: return {'error': 'Already published'} - if not alert: - url = urljoin(self.root_url, 'events/publish/{}'.format(event_id)) - else: - url = urljoin(self.root_url, 'events/alert/{}'.format(event_id)) - response = self.__prepare_request('POST', url) - return self._check_response(response) + return self.fast_publish(event_id, alert) def change_threat_level(self, event, threat_level_id): """Change the threat level of an event""" From 0f21a561b0a9e21550ff6d3fcc29cd2adaf2cfa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 15 Nov 2017 09:39:59 +0100 Subject: [PATCH 022/125] chg: Allow to add multiple attribute of the same type --- examples/add_generic_object.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) mode change 100644 => 100755 examples/add_generic_object.py diff --git a/examples/add_generic_object.py b/examples/add_generic_object.py old mode 100644 new mode 100755 index 308a1a3..4954644 --- a/examples/add_generic_object.py +++ b/examples/add_generic_object.py @@ -1,24 +1,35 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + import json from pymisp import PyMISP from pymisp.tools.abstractgenerator import AbstractMISPObjectGenerator from keys import misp_url, misp_key, misp_verifycert import argparse +""" +Sample usage: +./add_generic_object.py -e 5065 -t email -l '[{"to": "undisclosed@ppp.com"}, {"to": "second.to@mail.com"}]' +""" + + class GenericObject(AbstractMISPObjectGenerator): - def __init__(self, type, data_dict): + def __init__(self, type, attr_list): super(GenericObject, self).__init__(type) - self.__data = data_dict + self.__data = attr_list self.generate_attributes() def generate_attributes(self): - for key, value in self.__data.items(): - self.add_attribute(key, value=value) + for attribute in self.__data: + for key, value in attribute.items(): + self.add_attribute(key, value=value) + if __name__ == '__main__': parser = argparse.ArgumentParser(description='Create a MISP Object selectable by type starting from a dictionary') parser.add_argument("-e", "--event", required=True, help="Event ID to update") parser.add_argument("-t", "--type", required=True, help="Type of the generic object") - parser.add_argument("-d", "--dict", required=True, help="Dict ") + parser.add_argument("-l", "--attr_list", required=True, help="List of attributes") args = parser.parse_args() pymisp = PyMISP(misp_url, misp_key, misp_verifycert) @@ -29,5 +40,5 @@ if __name__ == '__main__': print ("Template for type %s not found! Valid types are: %s" % (args.type, valid_types)) exit() - misp_object = GenericObject(args.type.replace("|", "-"), json.loads(args.dict)) + misp_object = GenericObject(args.type.replace("|", "-"), json.loads(args.attr_list)) r = pymisp.add_object(args.event, template_id, misp_object) From bfe9867b2eada047b928e702ad653254b62be471 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 15 Nov 2017 17:37:17 +0100 Subject: [PATCH 023/125] chg: Add a generic MISP object generator --- examples/add_generic_object.py | 18 +++--------------- pymisp/mispevent.py | 1 + pymisp/tools/__init__.py | 1 + pymisp/tools/genericgenerator.py | 16 ++++++++++++++++ 4 files changed, 21 insertions(+), 15 deletions(-) create mode 100644 pymisp/tools/genericgenerator.py diff --git a/examples/add_generic_object.py b/examples/add_generic_object.py index 4954644..36e04cd 100755 --- a/examples/add_generic_object.py +++ b/examples/add_generic_object.py @@ -3,7 +3,7 @@ import json from pymisp import PyMISP -from pymisp.tools.abstractgenerator import AbstractMISPObjectGenerator +from pymisp.tools import GenericObjectGenerator from keys import misp_url, misp_key, misp_verifycert import argparse @@ -12,19 +12,6 @@ Sample usage: ./add_generic_object.py -e 5065 -t email -l '[{"to": "undisclosed@ppp.com"}, {"to": "second.to@mail.com"}]' """ - -class GenericObject(AbstractMISPObjectGenerator): - def __init__(self, type, attr_list): - super(GenericObject, self).__init__(type) - self.__data = attr_list - self.generate_attributes() - - def generate_attributes(self): - for attribute in self.__data: - for key, value in attribute.items(): - self.add_attribute(key, value=value) - - if __name__ == '__main__': parser = argparse.ArgumentParser(description='Create a MISP Object selectable by type starting from a dictionary') parser.add_argument("-e", "--event", required=True, help="Event ID to update") @@ -40,5 +27,6 @@ if __name__ == '__main__': print ("Template for type %s not found! Valid types are: %s" % (args.type, valid_types)) exit() - misp_object = GenericObject(args.type.replace("|", "-"), json.loads(args.attr_list)) + misp_object = GenericObjectGenerator(args.type.replace("|", "-")) + misp_object.generate_attributes(json.loads(args.attr_list)) r = pymisp.add_object(args.event, template_id, misp_object) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 645dc9b..32238f5 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -727,6 +727,7 @@ class MISPObject(AbstractMISP): attribute = MISPObjectAttribute(self.__definition['attributes'][object_relation]) else: # Woopsie, this object_relation is unknown, no sane defaults for you. + logger.warning("The template ({}) doesn't have the object_relation ({}) you're trying to add.".format(self.name, object_relation)) attribute = MISPObjectAttribute({}) else: attribute = MISPObjectAttribute({}) diff --git a/pymisp/tools/__init__.py b/pymisp/tools/__init__.py index 0412fde..e57c41a 100644 --- a/pymisp/tools/__init__.py +++ b/pymisp/tools/__init__.py @@ -6,3 +6,4 @@ from .elfobject import ELFObject, ELFSectionObject # noqa from .machoobject import MachOObject, MachOSectionObject # noqa from .create_misp_object import make_binary_objects # noqa from .abstractgenerator import AbstractMISPObjectGenerator # noqa +from .genericgenerator import GenericObjectGenerator # noqa diff --git a/pymisp/tools/genericgenerator.py b/pymisp/tools/genericgenerator.py new file mode 100644 index 0000000..06c688e --- /dev/null +++ b/pymisp/tools/genericgenerator.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from .abstractgenerator import AbstractMISPObjectGenerator + + +class GenericObjectGenerator(AbstractMISPObjectGenerator): + + def generate_attributes(self, attributes): + for attribute in attributes: + for object_relation, value in attribute.items(): + if isinstance(value, dict): + self.add_attribute(object_relation, **value) + else: + # In this case, we need a valid template, as all the other parameters will be pre-set. + self.add_attribute(object_relation, value=value) From 44831b1fdb86dad6205d68e35c7b107dac8d20ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 17 Nov 2017 16:51:46 +0100 Subject: [PATCH 024/125] chg: Add new objects: MISPUser and MISPOrganisation --- pymisp/api.py | 76 ++++++--------------------------------------- pymisp/mispevent.py | 20 ++++++++++++ 2 files changed, 30 insertions(+), 66 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 25c6bb4..4bbbf51 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -17,7 +17,7 @@ import zipfile from . import __version__ from .exceptions import PyMISPError, SearchError, NoURL, NoKey -from .mispevent import MISPEvent, MISPAttribute +from .mispevent import MISPEvent, MISPAttribute, MISPUser, MISPOrganisation from .abstract import MISPEncode logger = logging.getLogger('pymisp') @@ -1258,46 +1258,6 @@ class PyMISP(object): # ############## Users ################## - def _set_user_parameters(self, **kwargs): - user = {} - if kwargs.get('email'): - user['email'] = kwargs.get('email') - if kwargs.get('org_id'): - user['org_id'] = kwargs.get('org_id') - if kwargs.get('role_id'): - user['role_id'] = kwargs.get('role_id') - if kwargs.get('password'): - user['password'] = kwargs.get('password') - if kwargs.get('external_auth_required') is not None: - user['external_auth_required'] = kwargs.get('external_auth_required') - if kwargs.get('external_auth_key'): - user['external_auth_key'] = kwargs.get('external_auth_key') - if kwargs.get('enable_password') is not None: - user['enable_password'] = kwargs.get('enable_password') - if kwargs.get('nids_sid'): - user['nids_sid'] = kwargs.get('nids_sid') - if kwargs.get('server_id') is not None: - user['server_id'] = kwargs.get('server_id') - if kwargs.get('gpgkey'): - user['gpgkey'] = kwargs.get('gpgkey') - if kwargs.get('certif_public'): - user['certif_public'] = kwargs.get('certif_public') - if kwargs.get('autoalert') is not None: - user['autoalert'] = kwargs.get('autoalert') - if kwargs.get('contactalert') is not None: - user['contactalert'] = kwargs.get('contactalert') - if kwargs.get('disabled') is not None: - user['disabled'] = kwargs.get('disabled') - if kwargs.get('change_pw') is not None: - user['change_pw'] = kwargs.get('change_pw') - if kwargs.get('termsaccepted') is not None: - user['termsaccepted'] = kwargs.get('termsaccepted') - if kwargs.get('newsread') is not None: - user['newsread'] = kwargs.get('newsread') - if kwargs.get('authkey'): - user['authkey'] = kwargs.get('authkey') - return user - def get_users_list(self): url = urljoin(self.root_url, 'admin/users') response = self.__prepare_request('GET', url) @@ -1310,8 +1270,9 @@ class PyMISP(object): def add_user(self, email, org_id, role_id, **kwargs): url = urljoin(self.root_url, 'admin/users/add/') - new_user = self._set_user_parameters(**dict(email=email, org_id=org_id, role_id=role_id, **kwargs)) - response = self.__prepare_request('POST', url, json.dumps(new_user)) + new_user = MISPUser() + new_user.from_dict(email=email, org_id=org_id, role_id=role_id, **kwargs) + response = self.__prepare_request('POST', url, new_user.to_json()) return self._check_response(response) def add_user_json(self, json_file): @@ -1327,7 +1288,8 @@ class PyMISP(object): return self._check_response(response) def edit_user(self, user_id, **kwargs): - edit_user = self._set_user_parameters(**kwargs) + edit_user = MISPUser() + edit_user.from_dict(**kwargs) url = urljoin(self.root_url, 'admin/users/edit/{}'.format(user_id)) response = self.__prepare_request('POST', url, json.dumps(edit_user)) return self._check_response(response) @@ -1346,26 +1308,6 @@ class PyMISP(object): # ############## Organisations ################## - def _set_organisation_parameters(self, **kwargs): - organisation = {} - if kwargs.get('name'): - organisation['name'] = kwargs.get('name') - if kwargs.get('description'): - organisation['description'] = kwargs.get('description') - if kwargs.get('type'): - organisation['type'] = kwargs.get('type') - if kwargs.get('nationality'): - organisation['nationality'] = kwargs.get('nationality') - if kwargs.get('sector'): - organisation['sector'] = kwargs.get('sector') - if kwargs.get('uuid'): - organisation['uuid'] = kwargs.get('uuid') - if kwargs.get('contacts'): - organisation['contacts'] = kwargs.get('contacts') - if kwargs.get('local') is not None: - organisation['local'] = kwargs.get('local') - return organisation - def get_organisations_list(self, scope="local"): scope = scope.lower() if scope not in ["local", "external", "all"]: @@ -1380,7 +1322,8 @@ class PyMISP(object): return self._check_response(response) def add_organisation(self, name, **kwargs): - new_org = self._set_organisation_parameters(**dict(name=name, **kwargs)) + new_org = MISPOrganisation() + new_org.from_dict(name=name, **kwargs) if 'local' in new_org: if new_org.get('local') is False: if 'uuid' not in new_org: @@ -1402,7 +1345,8 @@ class PyMISP(object): return self._check_response(response) def edit_organisation(self, org_id, **kwargs): - edit_org = self._set_organisation_parameters(**kwargs) + edit_org = MISPOrganisation() + edit_org.from_dict(**kwargs) url = urljoin(self.root_url, 'admin/organisations/edit/{}'.format(org_id)) response = self.__prepare_request('POST', url, json.dumps(edit_org)) return self._check_response(response) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 32238f5..8bca094 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -595,6 +595,26 @@ class MISPObjectReference(AbstractMISP): setattr(self, k, v) +class MISPUser(AbstractMISP): + + def __init__(self): + super(MISPUser, self).__init__() + + def from_dict(self, **kwargs): + for k, v in kwargs.items(): + setattr(self, k, v) + + +class MISPOrganisation(AbstractMISP): + + def __init__(self): + super(MISPOrganisation, self).__init__() + + def from_dict(self, **kwargs): + for k, v in kwargs.items(): + setattr(self, k, v) + + class MISPObjectAttribute(MISPAttribute): def __init__(self, definition): From 89a55580146c0721b734f77ab2669ffe05fa20b9 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Sun, 19 Nov 2017 10:02:03 +0100 Subject: [PATCH 025/125] - Remove CIRCL reference from README.md - Updated 2 bad indentations where epydoc was Warning --- README.md | 4 ++-- pymisp/api.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index bf44f06..261534f 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ pip3 install pymisp ## Install the latest version from repo ``` -git clone https://github.com/CIRCL/PyMISP.git && cd PyMISP +git clone https://github.com/MISP/PyMISP.git && cd PyMISP pip3 install -I . ``` @@ -84,7 +84,7 @@ logging.basicConfig(level=logging.DEBUG, filename="debug.log", filemode='w', for Documentation can be generated with epydoc: ``` -epydoc --url https://github.com/CIRCL/PyMISP --graph all --name PyMISP --pdf pymisp -o doc +epydoc --url https://github.com/MISP/PyMISP --graph all --name PyMISP --pdf pymisp -o doc ``` ## Everything is a Mutable Mapping diff --git a/pymisp/api.py b/pymisp/api.py index 4bbbf51..e92f1de 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -68,9 +68,9 @@ class PyMISP(object): :param url: URL of the MISP instance you want to connect to :param key: API key of the user you want to use :param ssl: can be True or False (to check ot not the validity - of the certificate. Or a CA_BUNDLE in case of self - signed certiifcate (the concatenation of all the - *.crt of the chain) + of the certificate. Or a CA_BUNDLE in case of self + signed certiifcate (the concatenation of all the + *.crt of the chain) :param out_type: Type of object (json) NOTE: XML output isn't supported anymore, keeping the flag for compatibility reasons. :param debug: Write all the debug information to stderr :param proxies: Proxy dict as describes here: http://docs.python-requests.org/en/master/user/advanced/#proxies @@ -902,8 +902,8 @@ class PyMISP(object): analysis=None, attribute=None, org=None, async_callback=None, normalize=False): """Search only at the index level. Use ! infront of value as NOT, default OR If using async, give a callback that takes 2 args, session and response: - basic usage is - pymisp.search_index(..., async_callback=lambda ses,resp: print(resp.json())) + basic usage is + pymisp.search_index(..., async_callback=lambda ses,resp: print(resp.json())) :param published: Published (0,1) :param eventid: Evend ID(s) | str or list From 2e5fcf4e31e4f2bcfc20beba0edbd67f63d7a1bb Mon Sep 17 00:00:00 2001 From: c-goes Date: Mon, 20 Nov 2017 15:11:30 +0100 Subject: [PATCH 026/125] fixed typo --- pymisp/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/api.py b/pymisp/api.py index e92f1de..0956cd0 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -403,7 +403,7 @@ class PyMISP(object): :param alert: set to True by default (send alerting email) if False will not send alert :return publish status """ - if isinstance(event, int) or (isinstance(event, basestring) and event.is_digit()): + if isinstance(event, int) or (isinstance(event, basestring) and event.isdigit()): full_event = self._make_mispevent(self.get_event(event)) else: full_event = self._make_mispevent(event) From 4b5293ece19038c75eb8c8261dce0c3aa6313de1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 22 Nov 2017 10:30:06 +0100 Subject: [PATCH 027/125] chg: Change version number to master in the doc --- docs/source/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 4588cd1..bdaadc8 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -78,9 +78,9 @@ author = 'Raphaël Vinot' # built documents. # # The short X.Y version. -version = '2.4' +version = 'master' # The full version, including alpha/beta/rc tags. -release = '2.4.77' +release = 'master' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From 069023e8021102aed6ddcd7c2fa6b1e3efe25b7b Mon Sep 17 00:00:00 2001 From: 3c7 <3c7@posteo.de> Date: Wed, 22 Nov 2017 14:57:11 +0100 Subject: [PATCH 028/125] Adding multiple named attributes require a single POST request now --- pymisp/api.py | 88 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 63 insertions(+), 25 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 0956cd0..c7b43ef 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -449,35 +449,73 @@ class PyMISP(object): return self._check_response(response) # ##### File attributes ##### - def _send_attributes(self, event, attributes, proposal=False): - """Helper to add new attributes to an existing events""" - eventID_to_update = None - if isinstance(event, MISPEvent): - if hasattr(event, 'id'): - eventID_to_update = event.id - elif hasattr(event, 'uuid'): - eventID_to_update = event.uuid - elif isinstance(event, int) or (isinstance(event, str) and (event.isdigit() or self._valid_uuid(event))): - eventID_to_update = event - else: - e = MISPEvent(self.describe_types) - e.load(event) - if hasattr(e, 'id'): - eventID_to_update = e.id - elif hasattr(e, 'uuid'): - eventID_to_update = e.uuid - if eventID_to_update is None: - raise PyMISPError("Unable to find the ID of the event to update") + """ + Helper to add new attributes to an existing event, identified by an event object or an event id + + + :param event: EventID (int) or Event to alter + :param attributes: One or more attribute to add + :param proposal: True or False based on whether the attributes should be proposed or directly save + :type event: MISPEvent, int + :type attributes: MISPAttribute, list + :type proposal: bool + :return: list of responses + :rtype: list + """ + event_id = self._extract_event_id(event) + responses = [] + if not event_id: + raise PyMISPError("Unable to find the ID of the event to update.") if not attributes: return {'error': 'No attributes.'} - for a in attributes: - if proposal: - response = self.proposal_add(eventID_to_update, a) + + # Propals need to be posted in single requests + if proposal: + for a in attributes: + # proposal_add(...) returns a dict + responses.append(self.proposal_add(event_id, a)) + else: + url = urljoin(self.root_url, 'attributes/add/{}'.format(event_id)) + if isinstance(attributes, list): + values = [] + for a in attributes: + values.append(a['value']) + attributes[0]['value'] = values + data = attributes[0].to_json() else: - url = urljoin(self.root_url, 'attributes/add/{}'.format(eventID_to_update)) - response = self.__prepare_request('POST', url, a.to_json()) - return response + data = attributes.to_json() + # __prepare_request(...) returns a requests.Response Object + responses.append(self.__prepare_request('POST', url, data).json()) + return responses + + def _extract_event_id(self, event): + """ + Extracts the eventId from a given MISPEvent + + :param event: MISPEvent to extract the id from + :type event: MISPEvent + :return: EventId + :rtype: int + """ + event_id = None + if isinstance(event, MISPEvent): + if hasattr(event, 'id'): + event_id = event.id + elif hasattr(event, 'uuid'): + event_id = event.uuid + elif isinstance(event, int): + event_id = event + else: + e = MISPEvent(describe_types=self.describe_types) + e.load(event) + if hasattr(e, 'id'): + event_id = e.id + elif hasattr(e, 'uuid'): + event_id = e.uuid + return event_id + + def add_named_attribute(self, event, type_value, value, category=None, to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): """Add one or more attributes to an existing event""" From d4b8df380f7f62061264659194f1d8f9cc40fbb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 23 Nov 2017 10:17:36 +0100 Subject: [PATCH 029/125] chg: Bump misp-objects --- pymisp/data/misp-objects | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index 66c4578..56751a4 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit 66c4578f08efc6b92737f1667cbaf237149a8e46 +Subproject commit 56751a416e402e7227b1256e47b43cd48b1b4708 From 4416fe30b1802c541718d88590cad5a44cc97f00 Mon Sep 17 00:00:00 2001 From: c-goes Date: Thu, 23 Nov 2017 17:51:04 +0100 Subject: [PATCH 030/125] allow deletion of objects and object references --- pymisp/api.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pymisp/api.py b/pymisp/api.py index c7b43ef..65871ea 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1617,12 +1617,24 @@ class PyMISP(object): response = self.__prepare_request('POST', url, misp_object.to_json()) return self._check_response(response) + def delete_object(self, id): + """Deletes an object""" + url = urljoin(self.root_url, 'objects/delete/{}'.format(id)) + response = self.__prepare_request('POST', url) + return self._check_response(response) + def add_object_reference(self, misp_object_reference): """Add a reference to an object""" url = urljoin(self.root_url, 'object_references/add') response = self.__prepare_request('POST', url, misp_object_reference.to_json()) return self._check_response(response) + def delete_object_reference(self, id): + """Deletes a reference to an object""" + url = urljoin(self.root_url, 'object_references/delete/{}'.format(id)) + response = self.__prepare_request('POST', url) + return self._check_response(response) + def get_object_templates_list(self): """Returns the list of Object templates available on the MISP instance""" url = urljoin(self.root_url, 'objectTemplates') From 0875ad4a5fefbbbd193318dcf1169cf2f7100b26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 28 Nov 2017 11:54:08 +0100 Subject: [PATCH 031/125] chg: Add example file to push OpenIOC file to MISP chg: Add some imports in the tool's init file --- examples/openioc_to_misp.py | 27 +++++++++++++++++++++++++++ pymisp/tools/__init__.py | 1 + 2 files changed, 28 insertions(+) create mode 100755 examples/openioc_to_misp.py diff --git a/examples/openioc_to_misp.py b/examples/openioc_to_misp.py new file mode 100755 index 0000000..0011e9c --- /dev/null +++ b/examples/openioc_to_misp.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import argparse + +from pymisp import PyMISP +from keys import misp_url, misp_key, misp_verifycert +from pymisp.tools import load_openioc_file + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Convert an OpenIOC file to a MISPEvent. Optionnaly send it to MISP.') + parser.add_argument("-i", "--input", required=True, help="Input file") + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("-o", "--output", help="Output file") + group.add_argument("-m", "--misp", action='store_true', help="Create new event on MISP") + + args = parser.parse_args() + + misp_event = load_openioc_file(args.input) + + if args.misp: + pymisp = PyMISP(misp_url, misp_key, misp_verifycert, debug=True) + pymisp.add_event(misp_event) + else: + with open(args.output, 'w') as f: + f.write(misp_event.to_json()) diff --git a/pymisp/tools/__init__.py b/pymisp/tools/__init__.py index e57c41a..b551432 100644 --- a/pymisp/tools/__init__.py +++ b/pymisp/tools/__init__.py @@ -7,3 +7,4 @@ from .machoobject import MachOObject, MachOSectionObject # noqa from .create_misp_object import make_binary_objects # noqa from .abstractgenerator import AbstractMISPObjectGenerator # noqa from .genericgenerator import GenericObjectGenerator # noqa +from .openioc import load_openioc, load_openioc_file # noqa From 60f3111f47878f349f7a05f0f7df5b0e008465b1 Mon Sep 17 00:00:00 2001 From: Tristan METAYER Date: Wed, 29 Nov 2017 16:46:41 +0100 Subject: [PATCH 032/125] - Correction for 'last' param. 'last' gives the latest events that have been published - add get_events_last_modified() this function returns the modified events based on timestamp --- pymisp/api.py | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index c7b43ef..cb890a5 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1033,7 +1033,7 @@ class PyMISP(object): :param not_tags: Tags *not* to search for :param date_from: First date :param date_to: Last date - :param last: Last updated events (for example 5d or 12h or 30m) + :param last: Last published events (for example 5d or 12h or 30m) :param eventid: Last date :param withAttachments: return events with or without the attachments :param uuid: search by uuid @@ -1167,12 +1167,39 @@ class PyMISP(object): return True, details def download_last(self, last): - """Download the last updated events. + """Download the last published events. :param last: can be defined in days, hours, minutes (for example 5d or 12h or 30m) """ return self.search(last=last) + def get_events_last_modified(self, search_from, search_to=None): + """Download the last modified events. + + :param search_from: timestamp periode to start. can be defined as a date (2000-12-21) + :param search_to: tamestamp periode to stop + """ + + def checkIfDateAndConvert(d): + """Check if the format is a date otherwise we keep the temistamp""" + if d and len(d) == 10: + if d[4] == '-' and d[7] == '-': + return int(datetime.datetime.strptime(d, '%Y-%m-%d').strftime("%s")) + if d.isnumeric(): + return d + return False + + search_from = checkIfDateAndConvert(search_from) + search_to = checkIfDateAndConvert(search_to) + + if search_from: + if search_to: + return self.search(timestamp=[search_from, search_to]) + else: + return self.search(timestamp=search_from) + + return {'error': '"search_from" or "search_to" are not in a valid format (timestamp or date(2000-12-21'} + # ########## Tags ########## def get_all_tags(self, quiet=False): From c023c3bee1b11d1439b0d330e94e45629ee9c312 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 30 Nov 2017 15:10:30 +0100 Subject: [PATCH 033/125] Update doc badge links --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 261534f..26959e7 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ README ====== -[![Documentation Status](https://readthedocs.org/projects/pymisp/badge/?version=master)](http://pymisp.readthedocs.io/en/master/?badge=master) +[![Documentation Status](https://readthedocs.org/projects/pymisp/badge/?version=latest)](http://pymisp.readthedocs.io/?badge=latest) [![Build Status](https://travis-ci.org/MISP/PyMISP.svg?branch=master)](https://travis-ci.org/MISP/PyMISP) [![Coverage Status](https://coveralls.io/repos/github/MISP/PyMISP/badge.svg?branch=master)](https://coveralls.io/github/MISP/PyMISP?branch=master) From 298c0873cf96588574eb1098e0dd6d168e0d1bbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 30 Nov 2017 15:11:33 +0100 Subject: [PATCH 034/125] chg: update PDF link to doc --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 26959e7..4889a54 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ logging.basicConfig(level=logging.DEBUG, filename="debug.log", filemode='w', for ## Documentation -[PyMISP API documentation is available](https://media.readthedocs.org/pdf/pymisp/master/pymisp.pdf). +[PyMISP API documentation is available](https://media.readthedocs.org/pdf/pymisp/latest/pymisp.pdf). Documentation can be generated with epydoc: From 2e46f00055ca4415baf9d1e21f99a12e4c6f5a22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 1 Dec 2017 10:36:09 +0100 Subject: [PATCH 035/125] fix: Bump describeTypes.json Add testing --- pymisp/data/describeTypes.json | 21 +++++++++++++++++++-- tests/test_offline.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/pymisp/data/describeTypes.json b/pymisp/data/describeTypes.json index d5ee01c..3d8b20b 100644 --- a/pymisp/data/describeTypes.json +++ b/pymisp/data/describeTypes.json @@ -409,6 +409,14 @@ "default_category": "Network activity", "to_ids": 1 }, + "mac-address": { + "default_category": "Network activity", + "to_ids": 0 + }, + "mac-eui-64": { + "default_category": "Network activity", + "to_ids": 0 + }, "email-dst-display-name": { "default_category": "Payload delivery", "to_ids": 0 @@ -482,7 +490,7 @@ "to_ids": 0 }, "gender": { - "default_category": "", + "default_category": "Person", "to_ids": 0 }, "passport-number": { @@ -665,6 +673,8 @@ "ip-dst|port", "ip-src|port", "hostname|port", + "mac-address", + "mac-eui-64", "email-dst-display-name", "email-src-display-name", "email-header", @@ -777,6 +787,8 @@ "filename|imphash", "filename|impfuzzy", "filename|pehash", + "mac-address", + "mac-eui-64", "ip-src", "ip-dst", "ip-dst|port", @@ -930,6 +942,8 @@ "hostname", "domain", "domain|ip", + "mac-address", + "mac-eui-64", "email-dst", "url", "uri", @@ -964,7 +978,8 @@ "comment", "text", "x509-fingerprint-sha1", - "other" + "other", + "dns-soa-email" ], "External analysis": [ "md5", @@ -978,6 +993,8 @@ "ip-dst", "ip-dst|port", "ip-src|port", + "mac-address", + "mac-eui-64", "hostname", "domain", "domain|ip", diff --git a/tests/test_offline.py b/tests/test_offline.py index 70f8e3e..992fb80 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -241,5 +241,34 @@ class TestOffline(unittest.TestCase): return unittest.SkipTest() print(json_blob) + def test_describeTypes_sane_default(self, m): + sane_default = self.types['result']['sane_defaults'] + self.assertEqual(sorted(sane_default.keys()), sorted(self.types['result']['types'])) + + def test_describeTypes_categories(self, m): + category_type_mappings = self.types['result']['category_type_mappings'] + self.assertEqual(sorted(category_type_mappings.keys()), sorted(self.types['result']['categories'])) + + def test_describeTypes_types_in_categories(self, m): + category_type_mappings = self.types['result']['category_type_mappings'] + for category, types in category_type_mappings.items(): + existing_types = [t for t in types if t in self.types['result']['types']] + self.assertEqual(sorted(existing_types), sorted(types)) + + def test_describeTypes_types_have_category(self, m): + category_type_mappings = self.types['result']['category_type_mappings'] + all_types = set() + for category, types in category_type_mappings.items(): + all_types.update(types) + self.assertEqual(sorted(list(all_types)), sorted(self.types['result']['types'])) + + def test_describeTypes_sane_default_valid_category(self, m): + sane_default = self.types['result']['sane_defaults'] + categories = self.types['result']['categories'] + for t, sd in sane_default.items(): + self.assertTrue(sd['to_ids'] in [0, 1]) + self.assertTrue(sd['default_category'] in categories) + + if __name__ == '__main__': unittest.main() From 9c7923fe0a7459112e5fe967870b72d6ea6f2d95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 1 Dec 2017 12:01:42 +0100 Subject: [PATCH 036/125] new: Add get CSV method. --- examples/get_csv.py | 28 ++++++++++++++++++++++++++++ pymisp/api.py | 40 +++++++++++++++++++++++++++++++++++----- 2 files changed, 63 insertions(+), 5 deletions(-) create mode 100755 examples/get_csv.py diff --git a/examples/get_csv.py b/examples/get_csv.py new file mode 100755 index 0000000..33baf62 --- /dev/null +++ b/examples/get_csv.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import argparse + +from pymisp import PyMISP +from keys import misp_url, misp_key, misp_verifycert + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Get MISP stuff as CSV.') + parser.add_argument("-e", "--event_id", help="Event ID to fetch. Without it, it will fetch the whole database.") + parser.add_argument("-a", "--attribute", nargs='+', help="Attribute column names") + parser.add_argument("-o", "--object_attribute", nargs='+', help="Object attribute column names") + parser.add_argument("-t", "--misp_types", nargs='+', help="MISP types to fetch (ip-src, hostname, ...)") + parser.add_argument("-c", "--context", action='store_true', help="Add event level context (tags...)") + parser.add_argument("-i", "--ignore", action='store_true', help="Returns the attributes even if the event isn't published, or the attribute doesn't have the to_ids flag") + parser.add_argument("-f", "--outfile", help="Output file to write the CSV.") + + args = parser.parse_args() + pymisp = PyMISP(misp_url, misp_key, misp_verifycert, debug=True) + response = pymisp.get_csv(args.event_id, args.attribute, args.object_attribute, args.misp_types, args.context, args.ignore) + + if args.outfile: + with open(args.outfile, 'w') as f: + f.write(response) + else: + print(response) diff --git a/pymisp/api.py b/pymisp/api.py index c7b43ef..0ce9eaf 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -515,8 +515,6 @@ class PyMISP(object): event_id = e.uuid return event_id - - def add_named_attribute(self, event, type_value, value, category=None, to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): """Add one or more attributes to an existing event""" attributes = [] @@ -1329,7 +1327,7 @@ class PyMISP(object): edit_user = MISPUser() edit_user.from_dict(**kwargs) url = urljoin(self.root_url, 'admin/users/edit/{}'.format(user_id)) - response = self.__prepare_request('POST', url, json.dumps(edit_user)) + response = self.__prepare_request('POST', url, edit_user.to_json()) return self._check_response(response) def edit_user_json(self, json_file, user_id): @@ -1367,7 +1365,7 @@ class PyMISP(object): if 'uuid' not in new_org: raise PyMISPError('A remote org MUST have a valid uuid') url = urljoin(self.root_url, 'admin/organisations/add/') - response = self.__prepare_request('POST', url, json.dumps(new_org)) + response = self.__prepare_request('POST', url, new_org.to_json()) return self._check_response(response) def add_organisation_json(self, json_file): @@ -1386,7 +1384,7 @@ class PyMISP(object): edit_org = MISPOrganisation() edit_org.from_dict(**kwargs) url = urljoin(self.root_url, 'admin/organisations/edit/{}'.format(org_id)) - response = self.__prepare_request('POST', url, json.dumps(edit_org)) + response = self.__prepare_request('POST', url, edit_org.to_json()) return self._check_response(response) def edit_organisation_json(self, json_file, org_id): @@ -1550,6 +1548,38 @@ class PyMISP(object): def get_stix(self, **kwargs): return self.get_stix_event(**kwargs) + def get_csv(self, eventid=None, attributes=[], object_attributes=[], misp_types=[], context=False, ignore=False): + """Get MISP values in CSV format + :param eventid: The event ID to query + :param attributes: The column names to export from normal attributes (i.e. uuid, value, type, ...) + :param object_attributes: The column names to export from attributes within objects (i.e. uuid, value, type, ...) + :param misp_types: MISP types to get (i.e. ip-src, hostname, ...) + :param context: Add event level context (event_info,event_member_org,event_source_org,event_distribution,event_threat_level_id,event_analysis,event_date,event_tag) + :param ignore: Returns the attributes even if the event isn't published, or the attribute doesn't have the to_ids flag set + """ + url = urljoin(self.root_url, '/events/csv/download') + to_post = {} + if eventid: + to_post['eventid'] = eventid + if attributes: + to_post['attributes'] = attributes + if object_attributes: + to_post['object_attributes'] = object_attributes + if misp_types: + for t in misp_types: + if t not in self.types: + logger.warning('{} is not a valid type'.format(t)) + to_post['type'] = misp_types + if context: + to_post['includeContext'] = True + if ignore: + to_post['ignore'] = True + if to_post: + response = self.__prepare_request('POST', url, json.dumps(to_post), output_type='json') + else: + response = self.__prepare_request('POST', url, output_type='json') + return response.text + # ########################### # ######## Feed ######### # ########################### From eb5cf6c06ca29df482d2a5dc8040227a35b03404 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 1 Dec 2017 14:02:04 +0100 Subject: [PATCH 037/125] chg: Add validators for describeTypes on the live instance --- pymisp/api.py | 20 ++++++++++++-------- tests/test.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 0ce9eaf..b658c98 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -120,14 +120,7 @@ class PyMISP(object): raise PyMISPError('Unable to connect to MISP ({}). Please make sure the API key and the URL are correct (http/https is required): {}'.format(self.root_url, e)) try: - response = self.__prepare_request('GET', urljoin(self.root_url, 'attributes/describeTypes.json')) - describe_types = self._check_response(response) - if describe_types.get('error'): - for e in describe_types.get('error'): - raise PyMISPError('Failed: {}'.format(e)) - self.describe_types = describe_types['result'] - if not self.describe_types.get('sane_defaults'): - raise PyMISPError('The MISP server your are trying to reach is outdated (<2.4.52). Please use PyMISP v2.4.51.1 (pip install -I PyMISP==v2.4.51.1) and/or contact your administrator.') + self.describe_types = self.get_live_describe_types() except Exception: with open(os.path.join(self.resources_path, 'describeTypes.json'), 'r') as f: describe_types = json.load(f) @@ -138,6 +131,17 @@ class PyMISP(object): self.category_type_mapping = self.describe_types['category_type_mappings'] self.sane_default = self.describe_types['sane_defaults'] + def get_live_describe_types(self): + response = self.__prepare_request('GET', urljoin(self.root_url, 'attributes/describeTypes.json')) + describe_types = self._check_response(response) + if describe_types.get('error'): + for e in describe_types.get('error'): + raise PyMISPError('Failed: {}'.format(e)) + describe_types = describe_types['result'] + if not describe_types.get('sane_defaults'): + raise PyMISPError('The MISP server your are trying to reach is outdated (<2.4.52). Please use PyMISP v2.4.51.1 (pip install -I PyMISP==v2.4.51.1) and/or contact your administrator.') + return describe_types + def __prepare_request(self, request_type, url, data=None, background_callback=None, output_type='json'): if logger.isEnabledFor(logging.DEBUG): diff --git a/tests/test.py b/tests/test.py index 3910725..a8f22da 100755 --- a/tests/test.py +++ b/tests/test.py @@ -12,6 +12,7 @@ class TestBasic(unittest.TestCase): def setUp(self): self.maxDiff = None self.misp = PyMISP(url, key, True, 'json') + self.live_describe_types = self.misp.get_live_describe_types() def _clean_event(self, event): event['Event'].pop('orgc_id', None) @@ -253,5 +254,34 @@ class TestBasic(unittest.TestCase): def test_create_organisation(self): self.add_organisation() + def test_describeTypes_sane_default(self): + sane_default = self.live_describe_types['sane_defaults'] + self.assertEqual(sorted(sane_default.keys()), sorted(self.live_describe_types['types'])) + + def test_describeTypes_categories(self, m): + category_type_mappings = self.live_describe_types['category_type_mappings'] + self.assertEqual(sorted(category_type_mappings.keys()), sorted(self.live_describe_types['categories'])) + + def test_describeTypes_types_in_categories(self, m): + category_type_mappings = self.live_describe_types['category_type_mappings'] + for category, types in category_type_mappings.items(): + existing_types = [t for t in types if t in self.live_describe_types['types']] + self.assertEqual(sorted(existing_types), sorted(types)) + + def test_describeTypes_types_have_category(self, m): + category_type_mappings = self.live_describe_types['category_type_mappings'] + all_types = set() + for category, types in category_type_mappings.items(): + all_types.update(types) + self.assertEqual(sorted(list(all_types)), sorted(self.live_describe_types['types'])) + + def test_describeTypes_sane_default_valid_category(self, m): + sane_default = self.live_describe_types['sane_defaults'] + categories = self.live_describe_types['categories'] + for t, sd in sane_default.items(): + self.assertTrue(sd['to_ids'] in [0, 1]) + self.assertTrue(sd['default_category'] in categories) + + if __name__ == '__main__': unittest.main() From 20987f7f0f81665ae0d557f164ff71b84eb2802e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 1 Dec 2017 14:08:51 +0100 Subject: [PATCH 038/125] fix: Typo in live tests --- tests/test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test.py b/tests/test.py index a8f22da..52bed7a 100755 --- a/tests/test.py +++ b/tests/test.py @@ -258,24 +258,24 @@ class TestBasic(unittest.TestCase): sane_default = self.live_describe_types['sane_defaults'] self.assertEqual(sorted(sane_default.keys()), sorted(self.live_describe_types['types'])) - def test_describeTypes_categories(self, m): + def test_describeTypes_categories(self): category_type_mappings = self.live_describe_types['category_type_mappings'] self.assertEqual(sorted(category_type_mappings.keys()), sorted(self.live_describe_types['categories'])) - def test_describeTypes_types_in_categories(self, m): + def test_describeTypes_types_in_categories(self): category_type_mappings = self.live_describe_types['category_type_mappings'] for category, types in category_type_mappings.items(): existing_types = [t for t in types if t in self.live_describe_types['types']] self.assertEqual(sorted(existing_types), sorted(types)) - def test_describeTypes_types_have_category(self, m): + def test_describeTypes_types_have_category(self): category_type_mappings = self.live_describe_types['category_type_mappings'] all_types = set() for category, types in category_type_mappings.items(): all_types.update(types) self.assertEqual(sorted(list(all_types)), sorted(self.live_describe_types['types'])) - def test_describeTypes_sane_default_valid_category(self, m): + def test_describeTypes_sane_default_valid_category(self): sane_default = self.live_describe_types['sane_defaults'] categories = self.live_describe_types['categories'] for t, sd in sane_default.items(): From 9a2b82a2aeabfafacffe81a85fad67161328780a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 1 Dec 2017 14:35:19 +0100 Subject: [PATCH 039/125] chg: Add a way to check if the ACL is up-to-date --- pymisp/api.py | 5 +++++ tests/test.py | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/pymisp/api.py b/pymisp/api.py index b658c98..e69bb72 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -131,6 +131,11 @@ class PyMISP(object): self.category_type_mapping = self.describe_types['category_type_mappings'] self.sane_default = self.describe_types['sane_defaults'] + def get_live_query_acl(self): + """This should return an empty list, unless the ACL is outdated.""" + response = self.__prepare_request('GET', urljoin(self.root_url, 'events/queryACL.json')) + return self._check_response(response) + def get_live_describe_types(self): response = self.__prepare_request('GET', urljoin(self.root_url, 'attributes/describeTypes.json')) describe_types = self._check_response(response) diff --git a/tests/test.py b/tests/test.py index 52bed7a..4f61400 100755 --- a/tests/test.py +++ b/tests/test.py @@ -282,6 +282,10 @@ class TestBasic(unittest.TestCase): self.assertTrue(sd['to_ids'] in [0, 1]) self.assertTrue(sd['default_category'] in categories) + def test_live_acl(self): + query_acl = self.misp.get_live_query_acl() + self.assertEqual(query_acl, []) + if __name__ == '__main__': unittest.main() From 7ea7ca07d1942d25461699ce676e0f68147135b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 1 Dec 2017 14:40:52 +0100 Subject: [PATCH 040/125] fix: Typo in the tests --- tests/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test.py b/tests/test.py index 4f61400..a678191 100755 --- a/tests/test.py +++ b/tests/test.py @@ -284,7 +284,7 @@ class TestBasic(unittest.TestCase): def test_live_acl(self): query_acl = self.misp.get_live_query_acl() - self.assertEqual(query_acl, []) + self.assertEqual(query_acl['response'], []) if __name__ == '__main__': From f4439ae970921f9b734289764a66341bc752dc86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 1 Dec 2017 16:15:46 +0100 Subject: [PATCH 041/125] chg: Add live tests for recommended pymisp version and describeTypes up-to-date --- pymisp/api.py | 9 ++++++--- tests/test.py | 11 ++++++++++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index e69bb72..a2c29c0 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -122,9 +122,7 @@ class PyMISP(object): try: self.describe_types = self.get_live_describe_types() except Exception: - with open(os.path.join(self.resources_path, 'describeTypes.json'), 'r') as f: - describe_types = json.load(f) - self.describe_types = describe_types['result'] + self.describe_types = self.get_local_describe_types() self.categories = self.describe_types['categories'] self.types = self.describe_types['types'] @@ -136,6 +134,11 @@ class PyMISP(object): response = self.__prepare_request('GET', urljoin(self.root_url, 'events/queryACL.json')) return self._check_response(response) + def get_local_describe_types(self): + with open(os.path.join(self.resources_path, 'describeTypes.json'), 'r') as f: + describe_types = json.load(f) + return describe_types['result'] + def get_live_describe_types(self): response = self.__prepare_request('GET', urljoin(self.root_url, 'attributes/describeTypes.json')) describe_types = self._check_response(response) diff --git a/tests/test.py b/tests/test.py index a678191..acd5a77 100755 --- a/tests/test.py +++ b/tests/test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from pymisp import PyMISP +from pymisp import PyMISP, __version__ from keys import url, key import time @@ -282,10 +282,19 @@ class TestBasic(unittest.TestCase): self.assertTrue(sd['to_ids'] in [0, 1]) self.assertTrue(sd['default_category'] in categories) + def test_describeTypes_uptodate(self): + self.assertEqual(self.live_describe_types, self.misp.get_local_describe_types()) + def test_live_acl(self): query_acl = self.misp.get_live_query_acl() self.assertEqual(query_acl['response'], []) + def test_recommended_pymisp_version(self): + response = self.misp.get_recommended_api_version() + recommended_version_tup = tuple(int(x) for x in response['version'].split('.')) + pymisp_version_tup = tuple(int(x) for x in __version__.split('.'))[:3] + self.assertEqual(recommended_version_tup, pymisp_version_tup) + if __name__ == '__main__': unittest.main() From be65733d73249e8149d5b81805136c9bb5841a3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 1 Dec 2017 16:47:06 +0100 Subject: [PATCH 042/125] chg: Do not get the event from the server before publishing if PyMISP.publish gets an ID --- pymisp/api.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index a2c29c0..238158d 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -416,12 +416,12 @@ class PyMISP(object): :return publish status """ if isinstance(event, int) or (isinstance(event, basestring) and event.isdigit()): - full_event = self._make_mispevent(self.get_event(event)) + event_id = event else: full_event = self._make_mispevent(event) - event_id = full_event.id - if full_event.published: - return {'error': 'Already published'} + if full_event.published: + return {'error': 'Already published'} + event_id = full_event.id return self.fast_publish(event_id, alert) def change_threat_level(self, event, threat_level_id): From 119169d4eb8e45d8240802ff8612f7c1f75faa16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 1 Dec 2017 17:55:58 +0100 Subject: [PATCH 043/125] chg: Bump version to 2.4.83 --- pymisp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 443e9dc..cadbd96 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.4.82' +__version__ = '2.4.83' import sys import logging logger = logging.getLogger(__name__) From 2ec1f20bc6e6db13f85cdc169fc499c9571b1e24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 4 Dec 2017 14:11:45 +0100 Subject: [PATCH 044/125] chg: Bump misp-objects --- pymisp/data/misp-objects | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index 56751a4..c3f88d6 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit 56751a416e402e7227b1256e47b43cd48b1b4708 +Subproject commit c3f88d6901085c651132d4f40274a219deca5250 From b514600965e00123e7267e4552143b3bbe3fc679 Mon Sep 17 00:00:00 2001 From: c-goes Date: Mon, 4 Dec 2017 17:52:13 +0100 Subject: [PATCH 045/125] Make FileObject creation work if lief parsing fails --- pymisp/tools/create_misp_object.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pymisp/tools/create_misp_object.py b/pymisp/tools/create_misp_object.py index faa1258..95a43f1 100644 --- a/pymisp/tools/create_misp_object.py +++ b/pymisp/tools/create_misp_object.py @@ -62,8 +62,20 @@ def make_binary_objects(filepath=None, pseudofile=None, filename=None): logger.warning('Bad format: {}'.format(e)) except lief.bad_file as e: logger.warning('Bad file: {}'.format(e)) + except lief.conversion_error as e: + logger.warning('Conversion file: {}'.format(e)) + except lief.builder_error as e: + logger.warning('Builder file: {}'.format(e)) except lief.parser_error as e: logger.warning('Parser error: {}'.format(e)) + except lief.integrity_error as e: + logger.warning('Integrity error: {}'.format(e)) + except lief.pe_error as e: + logger.warning('PE error: {}'.format(e)) + except lief.type_error as e: + logger.warning('Type error: {}'.format(e)) + except lief.exception as e: + logger.warning('Lief exception: {}'.format(e)) except FileTypeNotImplemented as e: # noqa logger.warning(e) if not HAS_LIEF: From 42c5cc5e372d067102f47bd92424640f81368f38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 4 Dec 2017 18:43:44 +0100 Subject: [PATCH 046/125] chg: allow to pass a proxy to query VT --- pymisp/tools/vtreportobject.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pymisp/tools/vtreportobject.py b/pymisp/tools/vtreportobject.py index 927ca26..cc9e907 100644 --- a/pymisp/tools/vtreportobject.py +++ b/pymisp/tools/vtreportobject.py @@ -23,13 +23,14 @@ class VTReportObject(AbstractMISPObjectGenerator): :indicator: IOC to search VirusTotal for ''' - def __init__(self, apikey, indicator): + def __init__(self, apikey, indicator, vt_proxies=None): # PY3 way: # super().__init__("virustotal-report") super(VTReportObject, self).__init__("virustotal-report") indicator = indicator.strip() self._resource_type = self.__validate_resource(indicator) if self._resource_type: + self._proxies = vt_proxies self._report = self.__query_virustotal(apikey, indicator) self.generate_attributes() else: @@ -38,6 +39,9 @@ class VTReportObject(AbstractMISPObjectGenerator): # Mark as non_jsonable because we need to add the references manually after the object(s) have been created self.update_not_jsonable('ObjectReference') + def get_report(self): + return self._report + def generate_attributes(self): ''' Parse the VirusTotal report for relevant attributes ''' self.add_attribute("last-submission", value=self._report["scan_date"]) @@ -72,7 +76,10 @@ class VTReportObject(AbstractMISPObjectGenerator): url = "https://www.virustotal.com/vtapi/v2/{}/report".format(self._resource_type) params = {"apikey": apikey, "resource": resource} # for now assume we're using a public API key - we'll figure out private keys later - report = requests.get(url, params=params) + if self._proxies: + report = requests.get(url, params=params, proxies=self._proxies) + else: + report = requests.get(url, params=params) report = report.json() if report["response_code"] == 1: return report From 8a8b0c113d2046a507d1a84b5c290e575b79cc3b Mon Sep 17 00:00:00 2001 From: StrayLightning Date: Tue, 5 Dec 2017 17:38:19 +0000 Subject: [PATCH 047/125] Check for zero-length 500 response from the server and produce a suitable error message In experimenting with PyMISP I am triggering problems on the server I am using. Occasionally the server will return a 500 response with a message indicating an internal error, but more often than not it returns a 500 response with no contents, and _check_response falls over itself, generating hard-to-fathom exception from the json internals. This commit hardens _check_response by detecting zero-length responses and raising a suitable exception. Also fix a missing bracket in one of the subsequent exception strings. --- pymisp/api.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 948200e..fa2492a 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -223,13 +223,16 @@ class PyMISP(object): """Check if the response from the server is not an unexpected error""" errors = [] if response.status_code >= 500: - errors.append(response.json()) - logger.critical('Something bad happened on the server-side: {}'.format(response.json())) + if len(response.content) == 0: + raise PyMISPError('Something bad happened on the server-side and there was no content to be decoded') + else: + errors.append(response.json()) + logger.critical('Something bad happened on the server-side: {}'.format(response.json())) try: to_return = response.json() except ValueError: # It the server didn't return a JSON blob, we've a problem. - raise PyMISPError('Unknown error (something is very broken server-side: {}'.format(response.text)) + raise PyMISPError('Unknown error (something is very broken server-side: {})'.format(response.text)) if isinstance(to_return, (list, str)): to_return = {'response': to_return} From 748be6a0939a109df27e43e84185ea4e6a22c2d6 Mon Sep 17 00:00:00 2001 From: StrayLightning Date: Tue, 5 Dec 2017 18:04:11 +0000 Subject: [PATCH 048/125] Improve the exception message for a server 500+ response with no response content --- pymisp/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/api.py b/pymisp/api.py index fa2492a..9806f1d 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -224,7 +224,7 @@ class PyMISP(object): errors = [] if response.status_code >= 500: if len(response.content) == 0: - raise PyMISPError('Something bad happened on the server-side and there was no content to be decoded') + raise PyMISPError('Something bad happened on the server-side, but there was no response content to be decoded') else: errors.append(response.json()) logger.critical('Something bad happened on the server-side: {}'.format(response.json())) From afa21e26dc8839c2ef26d1929ae568cfd5285c3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 6 Dec 2017 10:45:51 +0100 Subject: [PATCH 049/125] chg: Update changelog --- CHANGELOG.txt | 107 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 8fc6177..30084d8 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -2,6 +2,111 @@ Changelog ========= +v2.4.83 (2017-12-06) +-------------------- + +New +~~~ +- Add get CSV method. [Raphaël Vinot] + +Changes +~~~~~~~ +- Allow to pass a proxy to query VT. [Raphaël Vinot] +- Bump misp-objects. [Raphaël Vinot] +- Bump version to 2.4.83. [Raphaël Vinot] +- Do not get the event from the server before publishing if + PyMISP.publish gets an ID. [Raphaël Vinot] +- Add live tests for recommended pymisp version and describeTypes up-to- + date. [Raphaël Vinot] +- Add a way to check if the ACL is up-to-date. [Raphaël Vinot] +- Add validators for describeTypes on the live instance. [Raphaël Vinot] +- Update PDF link to doc. [Raphaël Vinot] +- Add example file to push OpenIOC file to MISP. [Raphaël Vinot] + + chg: Add some imports in the tool's init file +- Bump misp-objects. [Raphaël Vinot] +- Change version number to master in the doc. [Raphaël Vinot] +- Add new objects: MISPUser and MISPOrganisation. [Raphaël Vinot] +- Add a generic MISP object generator. [Raphaël Vinot] +- Allow to add multiple attribute of the same type. [Raphaël Vinot] +- Add fast publish method. [Raphaël Vinot] + + Fix #86 +- Improve documentation. [Raphaël Vinot] + + Fix #121 + +Fix +~~~ +- Typo in the tests. [Raphaël Vinot] +- Typo in live tests. [Raphaël Vinot] +- Bump describeTypes.json. [Raphaël Vinot] + + Add testing + +Other +~~~~~ +- Merge pull request #147 from StrayLightning/master. [Raphaël Vinot] + + Check explicitly for a 500 response from the server with no response content +- Improve the exception message for a server 500+ response with no + response content. [StrayLightning] +- Check for zero-length 500 response from the server and produce a + suitable error message. [StrayLightning] + + In experimenting with PyMISP I am triggering problems on the server I + am using. Occasionally the server will return a 500 response with a + message indicating an internal error, but more often than not it returns + a 500 response with no contents, and _check_response falls over itself, + generating hard-to-fathom exception from the json internals. + + This commit hardens _check_response by detecting zero-length responses + and raising a suitable exception. + + Also fix a missing bracket in one of the subsequent exception strings. +- Merge pull request #146 from c-goes/lief_integrity_exception. [Raphaël + Vinot] + + Make FileObject creation work if lief parsing fails +- Make FileObject creation work if lief parsing fails. [c-goes] +- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot] +- Merge pull request #144 from c-goes/objects_delete. [Raphaël Vinot] + + allow deletion of objects and object references +- Allow deletion of objects and object references. [c-goes] +- Update doc badge links. [Raphaël Vinot] +- Merge pull request #143 from 3c7/feature/send_attributes. [Raphaël + Vinot] + + Adding multiple named attributes (without proposal) require a single POST request now +- Adding multiple named attributes require a single POST request now. + [3c7] +- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot] +- Merge pull request #142 from c-goes/master. [Raphaël Vinot] + + replaced is_digit() with isdigit() +- Fixed typo. [c-goes] +- Merge remote-tracking branch 'upstream/master' [c-goes] +- Merge pull request #141 from SteveClement/master. [Raphaël Vinot] + + Remove CIRCL repo references from README.md & fix epydoc +- - Remove CIRCL reference from README.md - Updated 2 bad indentations + where epydoc was Warning. [Steve Clement] +- Merge remote-tracking branch 'upstream/master' [c-goes] +- Merge branch 'master' of https://github.com/MISP/PyMISP into + messageidtype. [c-goes] +- Added default_category for email-message-id. [c-goes] +- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot] +- Merge branch 'feature/feedgenerator_rework' [iglocska] +- Merge branch 'master' of https://github.com/MISP/PyMISP into + feature/feedgenerator_rework. [iglocska] +- Rework of the feed generator. [iglocska] + + - use objects, attribute tags and object references correctly + - generate quickhashlist for fast lookups / future MISP caching mechanism + - saner structure (herp-a-derp) + + v2.4.82 (2017-11-09) -------------------- @@ -13,6 +118,8 @@ New Changes ~~~~~~~ +- Bump PyMISP version. [Raphaël Vinot] +- Bump CHANGELOG. [Raphaël Vinot] - Bump misp-objects. [Raphaël Vinot] - Update readme for new logging system. [Raphaël Vinot] - Small improvments in the logging system. [Raphaël Vinot] From 66ccf54c127d25fcd41ea735be7f0e56d045a19c Mon Sep 17 00:00:00 2001 From: Eric Jaw Date: Wed, 6 Dec 2017 11:07:36 -0500 Subject: [PATCH 050/125] fix: Typo in error output text description --- examples/get.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/get.py b/examples/get.py index d2be085..80e5270 100755 --- a/examples/get.py +++ b/examples/get.py @@ -39,7 +39,7 @@ if __name__ == '__main__': args = parser.parse_args() if args.output is not None and os.path.exists(args.output): - print('Output file already exists, abord.') + print('Output file already exists, abort.') exit(0) misp = init(misp_url, misp_key) From ab3a6bb4042e634e34b7b30d8fa63c6122c15eb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sat, 9 Dec 2017 11:52:26 +0100 Subject: [PATCH 051/125] chg: Bump misp-objects --- pymisp/data/misp-objects | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index c3f88d6..4eac353 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit c3f88d6901085c651132d4f40274a219deca5250 +Subproject commit 4eac3539c479e148088578dc2382c0c637c53944 From 52e079fea2923d6a5bff583dea2f5efa9bfda177 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sat, 9 Dec 2017 13:12:04 +0100 Subject: [PATCH 052/125] chg: Allow to pass a pseudofile to LIEF --- pymisp/tools/create_misp_object.py | 7 +++++-- tests/test_offline.py | 29 +++++++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/pymisp/tools/create_misp_object.py b/pymisp/tools/create_misp_object.py index 95a43f1..c6a4dea 100644 --- a/pymisp/tools/create_misp_object.py +++ b/pymisp/tools/create_misp_object.py @@ -49,9 +49,12 @@ def make_macho_objects(lief_parsed, misp_file): def make_binary_objects(filepath=None, pseudofile=None, filename=None): misp_file = FileObject(filepath=filepath, pseudofile=pseudofile, filename=filename) - if HAS_LIEF and filepath: + if HAS_LIEF and filepath or (pseudofile and filename): try: - lief_parsed = lief.parse(filepath) + if filepath: + lief_parsed = lief.parse(filepath=filepath) + else: + lief_parsed = lief.parse(raw=pseudofile.getvalue(), name=filename) if isinstance(lief_parsed, lief.PE.Binary): return make_pe_objects(lief_parsed, misp_file) elif isinstance(lief_parsed, lief.ELF.Binary): diff --git a/tests/test_offline.py b/tests/test_offline.py index 992fb80..b4c1644 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -5,6 +5,7 @@ import unittest import requests_mock import json import os +from io import BytesIO import pymisp as pm from pymisp import PyMISP @@ -210,9 +211,12 @@ class TestOffline(unittest.TestCase): p.add_internal_other(evt, 'foobar') p.add_attachment(evt, "testFile") - def make_objects(self, path): + def make_objects(self, path=None, pseudofile=None, filename=None): to_return = {'objects': [], 'references': []} - fo, peo, seos = make_binary_objects(path) + if path: + fo, peo, seos = make_binary_objects(path) + else: + fo, peo, seos = make_binary_objects(pseudofile=pseudofile, filename=filename) if seos: for s in seos: @@ -229,8 +233,29 @@ class TestOffline(unittest.TestCase): to_return['objects'].append(fo) if fo.ObjectReference: to_return['references'] += fo.ObjectReference + + # Remove UUIDs for comparing the objects. + for o in to_return['objects']: + o.pop('uuid') + for o in to_return['references']: + o.pop('referenced_uuid') + o.pop('object_uuid') return json.dumps(to_return, cls=MISPEncode) + def test_objects_pseudofile(self, m): + paths = ['cmd.exe', 'tmux', 'MachO-OSX-x64-ls'] + try: + for path in paths: + with open(os.path.join('tests', 'viper-test-files', 'test_files', path), 'rb') as f: + pseudo = BytesIO(f.read()) + json_blob = self.make_objects(pseudofile=pseudo, filename=path) + # Compare pseudo file / path + filepath_blob = self.make_objects(os.path.join('tests', 'viper-test-files', 'test_files', path)) + self.assertEqual(json_blob, filepath_blob) + except IOError: # Can be replaced with FileNotFoundError when support for python 2 is dropped + return unittest.SkipTest() + print(json_blob) + def test_objects(self, m): paths = ['cmd.exe', 'tmux', 'MachO-OSX-x64-ls'] try: From 7946e2ef8350b654100d46faf1e40c9569727372 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sat, 9 Dec 2017 13:35:44 +0100 Subject: [PATCH 053/125] fix: Disable pseudofile support in py2, skip tests. --- pymisp/tools/create_misp_object.py | 10 ++++++++-- tests/test_offline.py | 3 +++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/pymisp/tools/create_misp_object.py b/pymisp/tools/create_misp_object.py index c6a4dea..989da93 100644 --- a/pymisp/tools/create_misp_object.py +++ b/pymisp/tools/create_misp_object.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +import six + from . import FileObject, PEObject, ELFObject, MachOObject from ..exceptions import MISPObjectException import logging @@ -54,7 +56,11 @@ def make_binary_objects(filepath=None, pseudofile=None, filename=None): if filepath: lief_parsed = lief.parse(filepath=filepath) else: - lief_parsed = lief.parse(raw=pseudofile.getvalue(), name=filename) + if six.PY2: + logger.critical('Pseudofile is not supported in python2. Just update.') + lief_parsed = None + else: + lief_parsed = lief.parse(raw=pseudofile.getvalue(), name=filename) if isinstance(lief_parsed, lief.PE.Binary): return make_pe_objects(lief_parsed, misp_file) elif isinstance(lief_parsed, lief.ELF.Binary): @@ -79,7 +85,7 @@ def make_binary_objects(filepath=None, pseudofile=None, filename=None): logger.warning('Type error: {}'.format(e)) except lief.exception as e: logger.warning('Lief exception: {}'.format(e)) - except FileTypeNotImplemented as e: # noqa + except FileTypeNotImplemented as e: logger.warning(e) if not HAS_LIEF: logger.warning('Please install lief, documentation here: https://github.com/lief-project/LIEF') diff --git a/tests/test_offline.py b/tests/test_offline.py index b4c1644..e2d0c9c 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -5,6 +5,7 @@ import unittest import requests_mock import json import os +import six from io import BytesIO import pymisp as pm @@ -243,6 +244,8 @@ class TestOffline(unittest.TestCase): return json.dumps(to_return, cls=MISPEncode) def test_objects_pseudofile(self, m): + if six.PY2: + return unittest.SkipTest() paths = ['cmd.exe', 'tmux', 'MachO-OSX-x64-ls'] try: for path in paths: From e7f395a92c0a267ec13532849788708ea31f8488 Mon Sep 17 00:00:00 2001 From: "Stefan Hagen (Individual)" Date: Mon, 11 Dec 2017 14:00:43 +0100 Subject: [PATCH 054/125] enhance coverage and fix en passant with focus on api --- pymisp/api.py | 7 +++++-- tests/test_offline.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 9806f1d..2aaa35f 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -189,8 +189,11 @@ class PyMISP(object): messages = [] if response.get('error'): if isinstance(response['error'], list): - for e in response['errors']: - messages.append(e['error']['value'][0]) + for e in response['error']: + if isinstance(e, dict): + messages.append(e['error']['value'][0]) + else: + messages.append(e) else: messages.append(['error']) elif response.get('errors'): diff --git a/tests/test_offline.py b/tests/test_offline.py index 992fb80..50c1cae 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -269,6 +269,39 @@ class TestOffline(unittest.TestCase): self.assertTrue(sd['to_ids'] in [0, 1]) self.assertTrue(sd['default_category'] in categories) + def test_flatten_error_messages_singular(self, m): + self.initURI(m) + pymisp = PyMISP(self.domain, self.key) + error = pymisp.get(1) + response = self.auth_error_msg + response['error'] = ['foo', 'bar', 'baz'] + messages = pymisp.flatten_error_messages(response) + self.assertEqual(["foo", "bar", "baz"], messages) + + def test_flatten_error_messages_plural(self, m): + self.initURI(m) + pymisp = PyMISP(self.domain, self.key) + error = pymisp.get(1) + response = self.auth_error_msg + response['errors'] = {'foo': 42, 'bar': False, 'baz': ['oo', 'ka']} + messages = pymisp.flatten_error_messages(response) + self.assertEqual(['42 (foo)', 'False (bar)', 'oo', 'ka'], messages) + + def test_flatten_error_messages_nested(self, m): + self.initURI(m) + pymisp = PyMISP(self.domain, self.key) + error = pymisp.get(1) + response = self.auth_error_msg + response['errors'] = { + 'fo': {'o': 42}, 'ba': {'r': True}, 'b': {'a': ['z']}, 'd': {'e': {'e': ['p']}}} + messages = pymisp.flatten_error_messages(response) + self.assertEqual(['Error in o: 42', 'Error in r: True', 'Error in a: z', "Error in e: {'e': ['p']}"], messages) + + def test_test_connection(self, m): + self.initURI(m) + pymisp = PyMISP(self.domain, self.key) + self.assertTrue(pymisp.test_connection()) + if __name__ == '__main__': unittest.main() From 635c02eadce9d1b834a534e2e4367976cc0dcc39 Mon Sep 17 00:00:00 2001 From: "Stefan Hagen (Individual)" Date: Mon, 11 Dec 2017 14:18:06 +0100 Subject: [PATCH 055/125] changed asserts from dict usecases to set comparison to workaround non 3.6 behavior --- tests/test_offline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_offline.py b/tests/test_offline.py index 7a4e563..91d767d 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -313,7 +313,7 @@ class TestOffline(unittest.TestCase): response = self.auth_error_msg response['errors'] = {'foo': 42, 'bar': False, 'baz': ['oo', 'ka']} messages = pymisp.flatten_error_messages(response) - self.assertEqual(['42 (foo)', 'False (bar)', 'oo', 'ka'], messages) + self.assertEqual(set(['42 (foo)', 'False (bar)', 'oo', 'ka']), set(messages)) def test_flatten_error_messages_nested(self, m): self.initURI(m) @@ -323,7 +323,7 @@ class TestOffline(unittest.TestCase): response['errors'] = { 'fo': {'o': 42}, 'ba': {'r': True}, 'b': {'a': ['z']}, 'd': {'e': {'e': ['p']}}} messages = pymisp.flatten_error_messages(response) - self.assertEqual(['Error in o: 42', 'Error in r: True', 'Error in a: z', "Error in e: {'e': ['p']}"], messages) + self.assertEqual(set(['Error in o: 42', 'Error in r: True', 'Error in a: z', "Error in e: {'e': ['p']}"]), set(messages)) def test_test_connection(self, m): self.initURI(m) From 5cbcc09d7d9ee3bfad1b761984aa58fd957bfc38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 11 Dec 2017 15:01:25 +0100 Subject: [PATCH 056/125] new: Add method to get all the events modified in an interval --- pymisp/api.py | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index f89fc23..6b265e9 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -6,6 +6,7 @@ import sys import json import datetime +from dateutil.parser import parse import os import base64 import re @@ -1051,7 +1052,7 @@ class PyMISP(object): :param withAttachments: return events with or without the attachments :param uuid: search by uuid :param publish_timestamp: the publish timestamp - :param timestamp: the creation timestamp + :param timestamp: the timestamp of the last modification. Can be a list (from->to) :param enforceWarninglist: Enforce the warning lists :param searchall: full text search on the database :param metadata: return only metadata if True @@ -1186,32 +1187,33 @@ class PyMISP(object): """ return self.search(last=last) + def _string_to_timestamp(self, date_string): + pydate = parse(date_string) + if sys.version_info >= (3, 3): + # Sane python version + timestamp = pydate.timestamp() + else: + # Whatever + from datetime import timezone # Only for Python < 3.3 + timestamp = (pydate - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds() + return timestamp + def get_events_last_modified(self, search_from, search_to=None): """Download the last modified events. - :param search_from: timestamp periode to start. can be defined as a date (2000-12-21) - :param search_to: tamestamp periode to stop + :param search_from: Beginning of the interval. Can be either a timestamp, or a date (2000-12-21) + :param search_to: End of the interval. Can be either a timestamp, or a date (2000-12-21) """ - - def checkIfDateAndConvert(d): - """Check if the format is a date otherwise we keep the temistamp""" - if d and len(d) == 10: - if d[4] == '-' and d[7] == '-': - return int(datetime.datetime.strptime(d, '%Y-%m-%d').strftime("%s")) - if d.isnumeric(): - return d - return False - search_from = checkIfDateAndConvert(search_from) - search_to = checkIfDateAndConvert(search_to) + search_from = self._string_to_timestamp(search_from) - if search_from: - if search_to: - return self.search(timestamp=[search_from, search_to]) - else: - return self.search(timestamp=search_from) + if search_to is not None: + search_to = self._string_to_timestamp(search_to) + to_search = [search_from, search_to] + else: + to_search = search_from - return {'error': '"search_from" or "search_to" are not in a valid format (timestamp or date(2000-12-21'} + return self.search(timestamp=to_search) # ########## Tags ########## From f68faf7c6193063add0c3f0f81ac3b99bb490719 Mon Sep 17 00:00:00 2001 From: "Stefan Hagen (Individual)" Date: Mon, 11 Dec 2017 15:27:50 +0100 Subject: [PATCH 057/125] further tests added (for public methods) --- tests/test_offline.py | 65 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/tests/test_offline.py b/tests/test_offline.py index 91d767d..15f99b0 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -51,6 +51,13 @@ class TestOffline(unittest.TestCase): m.register_uri('DELETE', self.domain + 'events/3', json={'errors': ['Invalid event'], 'message': 'Invalid event', 'name': 'Invalid event', 'url': '/events/3'}) m.register_uri('GET', self.domain + 'attributes/delete/2', json={'message': 'Attribute deleted.'}) m.register_uri('POST', self.domain + 'events/index', json=self.search_index_result) + m.register_uri('POST', self.domain + 'attributes/edit/' + self.key, json={}) + m.register_uri('GET', self.domain + 'shadow_attributes/view/None', json={}) + m.register_uri('GET', self.domain + 'shadow_attributes/view/1', json={}) + m.register_uri('POST', self.domain + 'events/freeTextImport/1', json={}) + m.register_uri('POST', self.domain + 'attributes/restSearch', json={}) + m.register_uri('POST', self.domain + 'attributes/downloadSample', json={}) + m.register_uri('GET', self.domain + 'tags', json={'Tag': 'foo'}) def test_getEvent(self, m): self.initURI(m) @@ -330,6 +337,64 @@ class TestOffline(unittest.TestCase): pymisp = PyMISP(self.domain, self.key) self.assertTrue(pymisp.test_connection()) + def test_change_toids(self, m): + self.initURI(m) + pymisp = PyMISP(self.domain, self.key) + self.assertEqual({}, pymisp.change_toids(self.key, 1)) + + def test_change_toids_invalid(self, m): + self.initURI(m) + pymisp = PyMISP(self.domain, self.key) + try: + _ = pymisp.change_toids(self.key, 42) + self.assertFalse('Exception required for off domain value') + except Exception: + pass + + def test_proposal_view_default(self, m): + self.initURI(m) + pymisp = PyMISP(self.domain, self.key) + self.assertEqual({}, pymisp.proposal_view()) + + def test_proposal_view_event_1(self, m): + self.initURI(m) + pymisp = PyMISP(self.domain, self.key) + self.assertEqual({}, pymisp.proposal_view(event_id=1)) + + def test_proposal_view_event_overdetermined(self, m): + self.initURI(m) + pymisp = PyMISP(self.domain, self.key) + self.assertTrue(pymisp.proposal_view(event_id=1, proposal_id=42).get('error') is not None) + + def test_freetext(self, m): + self.initURI(m) + pymisp = PyMISP(self.domain, self.key) + self.assertEqual({}, pymisp.freetext(1, 'foo', adhereToWarninglists=True, distribution=42)) + + def test_freetext_offdomain(self, m): + self.initURI(m) + pymisp = PyMISP(self.domain, self.key) + try: + _ = pymisp.freetext(1, None, adhereToWarninglists='hard') + self.assertFalse('Exception required for off domain value') + except Exception: + pass + + def test_get_yara(self, m): + self.initURI(m) + pymisp = PyMISP(self.domain, self.key) + self.assertEqual((False, None), pymisp.get_yara(1)) + + def test_download_samples(self, m): + self.initURI(m) + pymisp = PyMISP(self.domain, self.key) + self.assertEqual((False, None), pymisp.download_samples()) + + def test_get_all_tags(self, m): + self.initURI(m) + pymisp = PyMISP(self.domain, self.key) + self.assertEqual({'Tag': 'foo'}, pymisp.get_all_tags()) + if __name__ == '__main__': unittest.main() From d19a10ddcc58e6c6aba248871c8e32268f016584 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 12 Dec 2017 17:34:09 +0100 Subject: [PATCH 058/125] chg: Make the library easier to use --- pymisp/abstract.py | 9 ++- pymisp/mispevent.py | 122 ++++++++++++++++++++++++----- pymisp/tools/create_misp_object.py | 23 +++--- pymisp/tools/elfobject.py | 14 ++-- pymisp/tools/fileobject.py | 6 +- pymisp/tools/machoobject.py | 14 ++-- pymisp/tools/peobject.py | 14 ++-- pymisp/tools/vtreportobject.py | 6 +- 8 files changed, 141 insertions(+), 67 deletions(-) diff --git a/pymisp/abstract.py b/pymisp/abstract.py index 151a3bd..0afc0f5 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -27,6 +27,9 @@ class AbstractMISP(collections.MutableMapping): __not_jsonable = [] + def __init__(self, **kwargs): + super(AbstractMISP, self).__init__() + def properties(self): to_return = [] for prop, value in vars(self).items(): @@ -67,7 +70,11 @@ class AbstractMISP(collections.MutableMapping): return json.dumps(self, cls=MISPEncode) def __getitem__(self, key): - return getattr(self, key) + try: + return getattr(self, key) + except AttributeError: + # Expected by pop and other dict-related methods + raise KeyError def __setitem__(self, key, value): setattr(self, key, value) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 8bca094..8c4adbf 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -304,7 +304,6 @@ class MISPEvent(AbstractMISP): describe_types = t['result'] self._types = describe_types['types'] - self.attributes = [] self.Tag = [] def _reinitialize_event(self): @@ -315,7 +314,6 @@ class MISPEvent(AbstractMISP): self.info = None self.published = False self.date = datetime.date.today() - self.attributes = [] # All other keys self.sig = None @@ -475,9 +473,9 @@ class MISPEvent(AbstractMISP): for a in kwargs.pop('Attribute'): attribute = MISPAttribute() attribute.set_all_values(**a) - if not hasattr(self, 'attributes'): - self.attributes = [] - self.attributes.append(attribute) + if not hasattr(self, 'Attribute'): + self.Attribute = [] + self.Attribute.append(attribute) # All other keys if kwargs.get('id'): @@ -518,22 +516,24 @@ class MISPEvent(AbstractMISP): to_return = super(MISPEvent, self).to_dict() if to_return.get('date'): to_return['date'] = self.date.isoformat() - if to_return.get('attributes'): - attributes = to_return.pop('attributes') - to_return['Attribute'] = [attribute.to_dict(with_timestamp) for attribute in attributes] - if to_return.get('RelatedEvent'): - to_return['RelatedEvent'] = [rel_event.to_dict() for rel_event in self.RelatedEvent] if with_timestamp and to_return.get('timestamp'): - to_return['timestamp'] = int(time.mktime(self.timestamp.timetuple())) + if sys.version_info >= (3, 3): + to_return['timestamp'] = self.timestamp.timestamp() + else: + from datetime import timezone # Only for Python < 3.3 + to_return['timestamp'] = (self.timestamp - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds() else: to_return.pop('timestamp', None) if with_timestamp and to_return.get('publish_timestamp'): - to_return['publish_timestamp'] = int(time.mktime(self.publish_timestamp.timetuple())) + if sys.version_info >= (3, 3): + to_return['publish_timestamp'] = self.publish_timestamp.timestamp() + else: + from datetime import timezone # Only for Python < 3.3 + to_return['publish_timestamp'] = (self.publish_timestamp - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds() else: to_return.pop('publish_timestamp', None) to_return = _int_to_str(to_return) to_return = {'Event': to_return} - jsonschema.validate(to_return, self.__json_schema) return to_return def add_tag(self, tag): @@ -580,6 +580,38 @@ class MISPEvent(AbstractMISP): self.attributes = [] self.attributes.append(attribute) + @property + def attributes(self): + return self.Attribute + + @property + def related_events(self): + return self.RelatedEvent + + @property + def objects(self): + return self.Object + + @property + def tags(self): + return self.Tag + + def get_object_by_id(self, object_id): + for obj in self.objects: + if hasattr(obj, 'id') and obj.id == object_id: + return obj + raise InvalidMISPObject('Object with {} does not exists in ths event'.format(object_id)) + + def add_object(self, obj): + if isinstance(obj, MISPObject): + self.Object.append(obj) + elif isinstance(obj, dict): + tmp_object = MISPObject(obj['name']) + tmp_object.from_dict(**obj) + self.Object.append(tmp_object) + else: + raise InvalidMISPObject("An object to add to an existing Event needs to be either a MISPObject, or a plain python dictionary") + class MISPObjectReference(AbstractMISP): @@ -644,8 +676,18 @@ class MISPObjectAttribute(MISPAttribute): class MISPObject(AbstractMISP): - def __init__(self, name, strict=False): - super(MISPObject, self).__init__() + def __init__(self, name, strict=False, standalone=False, default_attributes_paramaters={}, **kwargs): + ''' Master class representing a generic MISP object + :name: Name of the object + + :strict: Enforce validation with the object templates + + :standalone: The object will be pushed as directly on MISP, not as a part of an event. + In this case the ObjectReference needs to be pushed manually and cannot be in the JSON dump. + + :default_attributes_paramaters: Used as template for the attributes if they are not overwritten in add_attribute + ''' + super(MISPObject, self).__init__(**kwargs) self.__strict = strict self.name = name self.__misp_objects_path = os.path.join( @@ -666,11 +708,30 @@ class MISPObject(AbstractMISP): self.description = self.__definition['description'] self.template_version = self.__definition['version'] else: - # FIXME We need to set something for meta-category, template_uuid, description and template_version + # Then we have no meta-category, template_uuid, description and template_version pass self.uuid = str(uuid.uuid4()) - self.Attribute = [] + self.__fast_attribute_access = {} # Hashtable object_relation: [attributes] + self._default_attributes_paramaters = default_attributes_paramaters + if self._default_attributes_paramaters: + # Let's clean that up + self._default_attributes_paramaters.pop('value', None) # duh + self._default_attributes_paramaters.pop('uuid', None) # duh + self._default_attributes_paramaters.pop('id', None) # duh + self._default_attributes_paramaters.pop('object_id', None) # duh + self._default_attributes_paramaters.pop('type', None) # depends on the value + self._default_attributes_paramaters.pop('object_relation', None) # depends on the value + self._default_attributes_paramaters.pop('disable_correlation', None) # depends on the value + self._default_attributes_paramaters.pop('to_ids', None) # depends on the value + self._default_attributes_paramaters.pop('category', None) # depends on the value + self._default_attributes_paramaters.pop('deleted', None) # doesn't make sense to pre-set it + self._default_attributes_paramaters.pop('data', None) # in case the original in a sample or an attachment + self.distribution = self._default_attributes_paramaters.distribution self.ObjectReference = [] + self._standalone = standalone + if self._standalone: + # Mark as non_jsonable because we need to add the references manually after the object(s) have been created + self.update_not_jsonable('ObjectReference') def from_dict(self, **kwargs): if self.__known_template: @@ -696,6 +757,8 @@ class MISPObject(AbstractMISP): setattr(self, key, value) def to_dict(self, strict=False): + # Set the expected key (Attributes) + self.Attribute = self.attributes if strict or self.__strict and self.__known_template: self._validate() return super(MISPObject, self).to_dict() @@ -708,7 +771,7 @@ class MISPObject(AbstractMISP): def _validate(self): """Make sure the object we're creating has the required fields""" all_object_relations = [] - for a in self.Attribute: + for a in self.attributes: all_object_relations.append(a.object_relation) count_relations = dict(Counter(all_object_relations)) for key, counter in count_relations.items(): @@ -739,6 +802,22 @@ class MISPObject(AbstractMISP): relationship_type=relationship_type, comment=comment, **kwargs) self.ObjectReference.append(reference) + def get_attributes_by_relation(self, object_relation): + '''Returns the list of attributes with the given object relation in the object''' + return self.__fast_attribute_access.get(object_relation, []) + + def has_attributes_by_relation(self, list_of_relations): + '''True if all the relations in the list are defined in the object''' + return all(relation in self.__fast_attribute_access for relation in list_of_relations) + + @property + def attributes(self): + return [a for sublist in self.__fast_attribute_access.values() for a in sublist] + + @property + def references(self): + return self.ObjectReference + def add_attribute(self, object_relation, **value): if value.get('value') is None: return None @@ -751,6 +830,9 @@ class MISPObject(AbstractMISP): attribute = MISPObjectAttribute({}) else: attribute = MISPObjectAttribute({}) - attribute.from_dict(object_relation, **value) - self.Attribute.append(attribute) + # Overwrite the parameters of self._default_attributes_paramaters with the ones of value + attribute.from_dict(object_relation=object_relation, **dict(self._default_attributes_paramaters, **value)) + if not self.__fast_attribute_access.get(object_relation): + self.__fast_attribute_access[object_relation] = [] + self.__fast_attribute_access[object_relation].append(attribute) return attribute diff --git a/pymisp/tools/create_misp_object.py b/pymisp/tools/create_misp_object.py index 989da93..e12f76e 100644 --- a/pymisp/tools/create_misp_object.py +++ b/pymisp/tools/create_misp_object.py @@ -22,8 +22,8 @@ class FileTypeNotImplemented(MISPObjectException): pass -def make_pe_objects(lief_parsed, misp_file): - pe_object = PEObject(parsed=lief_parsed) +def make_pe_objects(lief_parsed, misp_file, standalone=True, default_attributes_paramaters={}): + pe_object = PEObject(parsed=lief_parsed, standalone=standalone, default_attributes_paramaters=default_attributes_paramaters) misp_file.add_reference(pe_object.uuid, 'included-in', 'PE indicators') pe_sections = [] for s in pe_object.sections: @@ -31,8 +31,8 @@ def make_pe_objects(lief_parsed, misp_file): return misp_file, pe_object, pe_sections -def make_elf_objects(lief_parsed, misp_file): - elf_object = ELFObject(parsed=lief_parsed) +def make_elf_objects(lief_parsed, misp_file, standalone=True, default_attributes_paramaters={}): + elf_object = ELFObject(parsed=lief_parsed, standalone=standalone, default_attributes_paramaters=default_attributes_paramaters) misp_file.add_reference(elf_object.uuid, 'included-in', 'ELF indicators') elf_sections = [] for s in elf_object.sections: @@ -40,8 +40,8 @@ def make_elf_objects(lief_parsed, misp_file): return misp_file, elf_object, elf_sections -def make_macho_objects(lief_parsed, misp_file): - macho_object = MachOObject(parsed=lief_parsed) +def make_macho_objects(lief_parsed, misp_file, standalone=True, default_attributes_paramaters={}): + macho_object = MachOObject(parsed=lief_parsed, standalone=standalone, default_attributes_paramaters=default_attributes_paramaters) misp_file.add_reference(macho_object.uuid, 'included-in', 'MachO indicators') macho_sections = [] for s in macho_object.sections: @@ -49,8 +49,9 @@ def make_macho_objects(lief_parsed, misp_file): return misp_file, macho_object, macho_sections -def make_binary_objects(filepath=None, pseudofile=None, filename=None): - misp_file = FileObject(filepath=filepath, pseudofile=pseudofile, filename=filename) +def make_binary_objects(filepath=None, pseudofile=None, filename=None, standalone=True, default_attributes_paramaters={}): + misp_file = FileObject(filepath=filepath, pseudofile=pseudofile, filename=filename, + standalone=standalone, default_attributes_paramaters=default_attributes_paramaters) if HAS_LIEF and filepath or (pseudofile and filename): try: if filepath: @@ -62,11 +63,11 @@ def make_binary_objects(filepath=None, pseudofile=None, filename=None): else: lief_parsed = lief.parse(raw=pseudofile.getvalue(), name=filename) if isinstance(lief_parsed, lief.PE.Binary): - return make_pe_objects(lief_parsed, misp_file) + return make_pe_objects(lief_parsed, misp_file, standalone, default_attributes_paramaters) elif isinstance(lief_parsed, lief.ELF.Binary): - return make_elf_objects(lief_parsed, misp_file) + return make_elf_objects(lief_parsed, misp_file, standalone, default_attributes_paramaters) elif isinstance(lief_parsed, lief.MachO.Binary): - return make_macho_objects(lief_parsed, misp_file) + return make_macho_objects(lief_parsed, misp_file, standalone, default_attributes_paramaters) except lief.bad_format as e: logger.warning('Bad format: {}'.format(e)) except lief.bad_file as e: diff --git a/pymisp/tools/elfobject.py b/pymisp/tools/elfobject.py index d9c9561..4dda680 100644 --- a/pymisp/tools/elfobject.py +++ b/pymisp/tools/elfobject.py @@ -24,7 +24,7 @@ except ImportError: class ELFObject(AbstractMISPObjectGenerator): - def __init__(self, parsed=None, filepath=None, pseudofile=None): + def __init__(self, parsed=None, filepath=None, pseudofile=None, standalone=True, **kwargs): if not HAS_PYDEEP: logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") if not HAS_LIEF: @@ -44,10 +44,8 @@ class ELFObject(AbstractMISPObjectGenerator): self.__elf = parsed else: raise InvalidMISPObject('Not a lief.ELF.Binary: {}'.format(type(parsed))) - super(ELFObject, self).__init__('elf') + super(ELFObject, self).__init__('elf', standalone=standalone, **kwargs) self.generate_attributes() - # Mark as non_jsonable because we need to add them manually - self.update_not_jsonable('ObjectReference') def generate_attributes(self): # General information @@ -60,7 +58,7 @@ class ELFObject(AbstractMISPObjectGenerator): if self.__elf.sections: pos = 0 for section in self.__elf.sections: - s = ELFSectionObject(section) + s = ELFSectionObject(section, self._standalone, default_attributes_paramaters=self._default_attributes_paramaters) self.add_reference(s.uuid, 'included-in', 'Section {} of ELF'.format(pos)) pos += 1 self.sections.append(s) @@ -69,15 +67,13 @@ class ELFObject(AbstractMISPObjectGenerator): class ELFSectionObject(AbstractMISPObjectGenerator): - def __init__(self, section): + def __init__(self, section, standalone=True, **kwargs): # Python3 way # super().__init__('pe-section') - super(ELFSectionObject, self).__init__('elf-section') + super(ELFSectionObject, self).__init__('elf-section', standalone=standalone, **kwargs) self.__section = section self.__data = bytes(self.__section.content) self.generate_attributes() - # Mark as non_jsonable because we need to add them manually - self.update_not_jsonable('ObjectReference') def generate_attributes(self): self.add_attribute('name', value=self.__section.name) diff --git a/pymisp/tools/fileobject.py b/pymisp/tools/fileobject.py index a12d38f..e9b05cd 100644 --- a/pymisp/tools/fileobject.py +++ b/pymisp/tools/fileobject.py @@ -28,7 +28,7 @@ except ImportError: class FileObject(AbstractMISPObjectGenerator): - def __init__(self, filepath=None, pseudofile=None, filename=None): + def __init__(self, filepath=None, pseudofile=None, filename=None, standalone=True, **kwargs): if not HAS_PYDEEP: logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") if not HAS_MAGIC: @@ -51,11 +51,9 @@ class FileObject(AbstractMISPObjectGenerator): raise InvalidMISPObject('File buffer (BytesIO) or a path is required.') # PY3 way: # super().__init__('file') - super(FileObject, self).__init__('file') + super(FileObject, self).__init__('file', standalone=standalone, **kwargs) self.__data = self.__pseudofile.getvalue() self.generate_attributes() - # Mark as non_jsonable because we need to add them manually - self.update_not_jsonable('ObjectReference') def generate_attributes(self): self.add_attribute('filename', value=self.__filename) diff --git a/pymisp/tools/machoobject.py b/pymisp/tools/machoobject.py index 6cf3fa2..fddda21 100644 --- a/pymisp/tools/machoobject.py +++ b/pymisp/tools/machoobject.py @@ -25,7 +25,7 @@ except ImportError: class MachOObject(AbstractMISPObjectGenerator): - def __init__(self, parsed=None, filepath=None, pseudofile=None): + def __init__(self, parsed=None, filepath=None, pseudofile=None, standalone=True, **kwargs): if not HAS_PYDEEP: logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") if not HAS_LIEF: @@ -47,10 +47,8 @@ class MachOObject(AbstractMISPObjectGenerator): raise InvalidMISPObject('Not a lief.MachO.Binary: {}'.format(type(parsed))) # Python3 way # super().__init__('elf') - super(MachOObject, self).__init__('macho') + super(MachOObject, self).__init__('macho', standalone=standalone, **kwargs) self.generate_attributes() - # Mark as non_jsonable because we need to add them manually - self.update_not_jsonable(['ObjectReference']) def generate_attributes(self): self.add_attribute('type', value=str(self.__macho.header.file_type).split('.')[1]) @@ -63,7 +61,7 @@ class MachOObject(AbstractMISPObjectGenerator): if self.__macho.sections: pos = 0 for section in self.__macho.sections: - s = MachOSectionObject(section) + s = MachOSectionObject(section, self._standalone, default_attributes_paramaters=self._default_attributes_paramaters) self.add_reference(s.uuid, 'included-in', 'Section {} of MachO'.format(pos)) pos += 1 self.sections.append(s) @@ -72,15 +70,13 @@ class MachOObject(AbstractMISPObjectGenerator): class MachOSectionObject(AbstractMISPObjectGenerator): - def __init__(self, section): + def __init__(self, section, standalone=True, **kwargs): # Python3 way # super().__init__('pe-section') - super(MachOSectionObject, self).__init__('macho-section') + super(MachOSectionObject, self).__init__('macho-section', standalone=standalone, **kwargs) self.__section = section self.__data = bytes(self.__section.content) self.generate_attributes() - # Mark as non_jsonable because we need to add them manually - self.update_not_jsonable(['ObjectReference']) def generate_attributes(self): self.add_attribute('name', value=self.__section.name) diff --git a/pymisp/tools/peobject.py b/pymisp/tools/peobject.py index 2e8bf4a..8f32426 100644 --- a/pymisp/tools/peobject.py +++ b/pymisp/tools/peobject.py @@ -25,7 +25,7 @@ except ImportError: class PEObject(AbstractMISPObjectGenerator): - def __init__(self, parsed=None, filepath=None, pseudofile=None): + def __init__(self, parsed=None, filepath=None, pseudofile=None, standalone=True, **kwargs): if not HAS_PYDEEP: logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") if not HAS_LIEF: @@ -47,10 +47,8 @@ class PEObject(AbstractMISPObjectGenerator): raise InvalidMISPObject('Not a lief.PE.Binary: {}'.format(type(parsed))) # Python3 way # super().__init__('pe') - super(PEObject, self).__init__('pe') + super(PEObject, self).__init__('pe', standalone=standalone, **kwargs) self.generate_attributes() - # Mark as non_jsonable because we need to add them manually - self.update_not_jsonable('ObjectReference') def _is_exe(self): if not self._is_dll() and not self._is_driver(): @@ -106,7 +104,7 @@ class PEObject(AbstractMISPObjectGenerator): if self.__pe.sections: pos = 0 for section in self.__pe.sections: - s = PESectionObject(section) + s = PESectionObject(section, self._standalone, default_attributes_paramaters=self._default_attributes_paramaters) self.add_reference(s.uuid, 'included-in', 'Section {} of PE'.format(pos)) if ((self.__pe.entrypoint >= section.virtual_address) and (self.__pe.entrypoint < (section.virtual_address + section.virtual_size))): @@ -119,15 +117,13 @@ class PEObject(AbstractMISPObjectGenerator): class PESectionObject(AbstractMISPObjectGenerator): - def __init__(self, section): + def __init__(self, section, standalone=True, **kwargs): # Python3 way # super().__init__('pe-section') - super(PESectionObject, self).__init__('pe-section') + super(PESectionObject, self).__init__('pe-section', standalone=standalone, **kwargs) self.__section = section self.__data = bytes(self.__section.content) self.generate_attributes() - # Mark as non_jsonable because we need to add them manually - self.update_not_jsonable('ObjectReference') def generate_attributes(self): self.add_attribute('name', value=self.__section.name) diff --git a/pymisp/tools/vtreportobject.py b/pymisp/tools/vtreportobject.py index cc9e907..69e856e 100644 --- a/pymisp/tools/vtreportobject.py +++ b/pymisp/tools/vtreportobject.py @@ -23,10 +23,10 @@ class VTReportObject(AbstractMISPObjectGenerator): :indicator: IOC to search VirusTotal for ''' - def __init__(self, apikey, indicator, vt_proxies=None): + def __init__(self, apikey, indicator, vt_proxies=None, standalone=True, **kwargs): # PY3 way: # super().__init__("virustotal-report") - super(VTReportObject, self).__init__("virustotal-report") + super(VTReportObject, self).__init__("virustotal-report", standalone=standalone, **kwargs) indicator = indicator.strip() self._resource_type = self.__validate_resource(indicator) if self._resource_type: @@ -36,8 +36,6 @@ class VTReportObject(AbstractMISPObjectGenerator): else: error_msg = "A valid indicator is required. (One of type url, md5, sha1, sha256). Received '{}' instead".format(indicator) raise InvalidMISPObject(error_msg) - # Mark as non_jsonable because we need to add the references manually after the object(s) have been created - self.update_not_jsonable('ObjectReference') def get_report(self): return self._report From 6db31b397a4955f4ce4e586542f7df173aa48c4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 13 Dec 2017 16:43:21 +0100 Subject: [PATCH 059/125] new: Add methods to get taxonomy(ies) Thanks to @truckydev --- pymisp/api.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pymisp/api.py b/pymisp/api.py index a0d285e..f59b502 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1550,6 +1550,18 @@ class PyMISP(object): response = self.__prepare_request('GET', url) return self._check_response(response)['Tag'] + # ############## Taxonomies ################## + + def get_taxonomies_list(self): + url = urljoin(self.root_url, '/taxonomies') + response = self.__prepare_request('GET', url) + return self._check_response(response) + + def get_taxonomy(self, taxonomy_id): + url = urljoin(self.root_url, '/taxonomies/view/{}'.format(taxonomy_id)) + response = self.__prepare_request('GET', url) + return self._check_response(response) + # ############################################## # ############### Non-JSON output ############## # ############################################## From f02f88907754b7950b53ad9e76a3ae7108e0d95b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 13 Dec 2017 17:09:46 +0100 Subject: [PATCH 060/125] chg: Bump version --- pymisp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index cadbd96..5de8bee 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.4.83' +__version__ = '2.4.84' import sys import logging logger = logging.getLogger(__name__) From 98d63be062831d26eee9cce009cbf32f13f4fbd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 13 Dec 2017 17:11:22 +0100 Subject: [PATCH 061/125] chg: bump Changelog --- CHANGELOG.txt | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 30084d8..bad9051 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -2,6 +2,59 @@ Changelog ========= +v2.4.84 (2017-12-13) +-------------------- + +New +~~~ +- Add methods to get taxonomy(ies) [Raphaël Vinot] + + Thanks to @truckydev +- Add method to get all the events modified in an interval. [Raphaël + Vinot] + +Changes +~~~~~~~ +- Bump version. [Raphaël Vinot] +- Make the library easier to use. [Raphaël Vinot] +- Allow to pass a pseudofile to LIEF. [Raphaël Vinot] +- Bump misp-objects. [Raphaël Vinot] +- Update changelog. [Raphaël Vinot] + +Fix +~~~ +- Disable pseudofile support in py2, skip tests. [Raphaël Vinot] +- Typo in error output text description. [Eric Jaw] + +Other +~~~~~ +- Merge pull request #151 from MISP/refactor. [Raphaël Vinot] + + chg: Make the library easier to use +- Merge pull request #150 from sdrees/first-friendly-contribution- + enhance-coverage. [Raphaël Vinot] + + First friendly contribution enhance coverage +- Further tests added (for public methods) [Stefan Hagen (Individual)] +- Changed asserts from dict usecases to set comparison to workaround non + 3.6 behavior. [Stefan Hagen (Individual)] +- Merge branch 'master' of https://github.com/MISP/PyMISP into first- + friendly-contribution-enhance-coverage. [Stefan Hagen (Individual)] +- Enhance coverage and fix en passant with focus on api. [Stefan Hagen + (Individual)] +- Merge branch 'truckydev-get_last_modified_event' [Raphaël Vinot] +- Merge branch 'get_last_modified_event' of + https://github.com/truckydev/PyMISP into truckydev- + get_last_modified_event. [Raphaël Vinot] +- - Correction for 'last' param. 'last' gives the latest events that + have been published - add get_events_last_modified() this function + returns the modified events based on timestamp. [Tristan METAYER] +- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot] +- Merge pull request #149 from naisanza/master. [Raphaël Vinot] + + fix: Typo in error output text description + + v2.4.83 (2017-12-06) -------------------- From 68afcf1c3893f7fbe5feb5ffc224dc7031b461f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 13 Dec 2017 17:12:35 +0100 Subject: [PATCH 062/125] chg: Bump misp-objects --- pymisp/data/misp-objects | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index 4eac353..de36d3b 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit 4eac3539c479e148088578dc2382c0c637c53944 +Subproject commit de36d3b735ad97bc6b03b87d9a1ecb3feec2a733 From f15341d1cf7c4d3ffaebaac39320aab9db8078d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 13 Dec 2017 18:00:31 +0100 Subject: [PATCH 063/125] chg: Bump describeTypes --- pymisp/data/describeTypes.json | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pymisp/data/describeTypes.json b/pymisp/data/describeTypes.json index 3d8b20b..6349f89 100644 --- a/pymisp/data/describeTypes.json +++ b/pymisp/data/describeTypes.json @@ -373,6 +373,14 @@ "default_category": "Network activity", "to_ids": 1 }, + "x509-fingerprint-md5": { + "default_category": "Network activity", + "to_ids": 1 + }, + "x509-fingerprint-sha256": { + "default_category": "Network activity", + "to_ids": 1 + }, "dns-soa-email": { "default_category": "Attribution", "to_ids": 0 @@ -664,6 +672,8 @@ "whois-registrar", "whois-creation-date", "x509-fingerprint-sha1", + "x509-fingerprint-md5", + "x509-fingerprint-sha256", "dns-soa-email", "size-in-bytes", "counter", @@ -816,6 +826,8 @@ "hex", "vulnerability", "x509-fingerprint-sha1", + "x509-fingerprint-md5", + "x509-fingerprint-sha256", "other", "hostname|port", "email-dst-display-name", @@ -875,6 +887,8 @@ "text", "hex", "x509-fingerprint-sha1", + "x509-fingerprint-md5", + "x509-fingerprint-sha256", "other", "cookie" ], @@ -921,6 +935,8 @@ "text", "hex", "x509-fingerprint-sha1", + "x509-fingerprint-md5", + "x509-fingerprint-sha256", "mobile-application-id", "other" ], @@ -978,6 +994,8 @@ "comment", "text", "x509-fingerprint-sha1", + "x509-fingerprint-md5", + "x509-fingerprint-sha256", "other", "dns-soa-email" ], @@ -1014,6 +1032,8 @@ "comment", "text", "x509-fingerprint-sha1", + "x509-fingerprint-md5", + "x509-fingerprint-sha256", "github-repository", "other", "cortex" From 78d9673e24e995ca22b786b1c6d74093aa9c760d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 13 Dec 2017 18:01:42 +0100 Subject: [PATCH 064/125] chg: Bump misp-objects --- pymisp/data/misp-objects | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index de36d3b..b85438f 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit de36d3b735ad97bc6b03b87d9a1ecb3feec2a733 +Subproject commit b85438fc45b212a21b72d6d2e0df619758fa1444 From 19a50a7ba7cf1ba4c932f1f19009fee253070288 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 14 Dec 2017 11:06:52 +0100 Subject: [PATCH 065/125] chg: Use new format for filtering. --- pymisp/api.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index f59b502..0030b0e 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1021,21 +1021,17 @@ class PyMISP(object): :param values: Values to search :param not_values: Values that should not be in the response """ - to_return = '' + to_return = [] if values is not None: - if not isinstance(values, list): + if isinstance(values, list): to_return += values else: - to_return += '&&'.join(values) + to_return.append(values) if not_values is not None: - if len(to_return) > 0: - to_return += '&&!' + if isinstance(not_values, list): + to_return += ['!{}'.format(v) for v in not_values] else: - to_return += '!' - if not isinstance(not_values, list): - to_return += not_values - else: - to_return += '&&!'.join(not_values) + to_return.append('!{}'.format(not_values)) return to_return def search(self, controller='events', async_callback=None, **kwargs): @@ -1068,7 +1064,7 @@ class PyMISP(object): # Event: array('value', 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'searchall', 'metadata', 'published'); # Attribute: array('value', 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'to_ids', 'deleted'); val = self.__prepare_rest_search(kwargs.pop('values', None), kwargs.pop('not_values', None)) - if len(val) != 0: + if val: query['value'] = val query['type'] = kwargs.pop('type_attribute', None) @@ -1076,7 +1072,7 @@ class PyMISP(object): query['org'] = kwargs.pop('org', None) tag = self.__prepare_rest_search(kwargs.pop('tags', None), kwargs.pop('not_tags', None)) - if len(tag) != 0: + if tag: query['tags'] = tag date_from = kwargs.pop('date_from', None) From 56c74f1f50047e294112ed71a7b07bcc4acf7ced Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 14 Dec 2017 16:12:54 +0100 Subject: [PATCH 066/125] chg: Add __repr__ methods --- pymisp/mispevent.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 8c4adbf..368ff07 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -147,6 +147,9 @@ class MISPAttribute(AbstractMISP): # to be deprecated self.from_dict(**kwargs) + def __repr__(self): + return '<{self.__class__.__name__}(self.type, self.value)'.format(self=self) + def from_dict(self, **kwargs): if kwargs.get('type') and kwargs.get('category'): if kwargs['type'] not in self.__category_type_mapping[kwargs['category']]: @@ -438,6 +441,9 @@ class MISPEvent(AbstractMISP): # to be deprecated self.from_dict(**kwargs) + def __repr__(self): + return '<{self.__class__.__name__}(self.info, self.date)'.format(self=self) + def from_dict(self, **kwargs): # Required value self.info = kwargs.pop('info', None) @@ -618,6 +624,9 @@ class MISPObjectReference(AbstractMISP): def __init__(self): super(MISPObjectReference, self).__init__() + def __repr__(self): + return '<{self.__class__.__name__}(self.object_uuid, self.referenced_uuid, self.relationship_type)'.format(self=self) + def from_dict(self, object_uuid, referenced_uuid, relationship_type, comment=None, **kwargs): self.object_uuid = object_uuid self.referenced_uuid = referenced_uuid @@ -653,6 +662,9 @@ class MISPObjectAttribute(MISPAttribute): super(MISPObjectAttribute, self).__init__() self.__definition = definition + def __repr__(self): + return '<{self.__class__.__name__}(self.object_relation, self.value)'.format(self=self) + def from_dict(self, object_relation, value, **kwargs): self.object_relation = object_relation self.value = value @@ -733,6 +745,9 @@ class MISPObject(AbstractMISP): # Mark as non_jsonable because we need to add the references manually after the object(s) have been created self.update_not_jsonable('ObjectReference') + def __repr__(self): + return '<{self.__class__.__name__}(self.name)'.format(self=self) + def from_dict(self, **kwargs): if self.__known_template: if kwargs.get('template_uuid') and kwargs['template_uuid'] != self.template_uuid: From 892ebbf07770abe8f31931a0d7dcd4f242769097 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 15 Dec 2017 17:34:54 +0100 Subject: [PATCH 067/125] chg: Add __repr__ methods (fix last commit) --- pymisp/api.py | 3 +++ pymisp/mispevent.py | 20 +++++++++++++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 0030b0e..a4d91fc 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -130,6 +130,9 @@ class PyMISP(object): self.category_type_mapping = self.describe_types['category_type_mappings'] self.sane_default = self.describe_types['sane_defaults'] + def __repr__(self): + return '<{self.__class__.__name__}(url={self.root_url})'.format(self=self) + def get_live_query_acl(self): """This should return an empty list, unless the ACL is outdated.""" response = self.__prepare_request('GET', urljoin(self.root_url, 'events/queryACL.json')) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 368ff07..c762bf7 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -148,7 +148,9 @@ class MISPAttribute(AbstractMISP): self.from_dict(**kwargs) def __repr__(self): - return '<{self.__class__.__name__}(self.type, self.value)'.format(self=self) + if hasattr(self, 'value'): + return '<{self.__class__.__name__}(type={self.type}, value={self.value})'.format(self=self) + return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) def from_dict(self, **kwargs): if kwargs.get('type') and kwargs.get('category'): @@ -442,7 +444,9 @@ class MISPEvent(AbstractMISP): self.from_dict(**kwargs) def __repr__(self): - return '<{self.__class__.__name__}(self.info, self.date)'.format(self=self) + if hasattr(self, 'info'): + return '<{self.__class__.__name__}(info={self.info}, date={self.date})'.format(self=self) + return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) def from_dict(self, **kwargs): # Required value @@ -625,7 +629,9 @@ class MISPObjectReference(AbstractMISP): super(MISPObjectReference, self).__init__() def __repr__(self): - return '<{self.__class__.__name__}(self.object_uuid, self.referenced_uuid, self.relationship_type)'.format(self=self) + if hasattr(self, 'referenced_uuid'): + return '<{self.__class__.__name__}(object_uuid={self.object_uuid}, referenced_uuid={self.referenced_uuid}, relationship_type={self.relationship_type})'.format(self=self) + return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) def from_dict(self, object_uuid, referenced_uuid, relationship_type, comment=None, **kwargs): self.object_uuid = object_uuid @@ -663,7 +669,9 @@ class MISPObjectAttribute(MISPAttribute): self.__definition = definition def __repr__(self): - return '<{self.__class__.__name__}(self.object_relation, self.value)'.format(self=self) + if hasattr(self, 'value'): + return '<{self.__class__.__name__}(object_relation={self.object_relation}, value={self.value})'.format(self=self) + return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) def from_dict(self, object_relation, value, **kwargs): self.object_relation = object_relation @@ -746,7 +754,9 @@ class MISPObject(AbstractMISP): self.update_not_jsonable('ObjectReference') def __repr__(self): - return '<{self.__class__.__name__}(self.name)'.format(self=self) + if hasattr(self, 'name'): + return '<{self.__class__.__name__}(name={self.name})'.format(self=self) + return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) def from_dict(self, **kwargs): if self.__known_template: From 04351dfa80d2ea4d3a264ee0eba1ab571f0ab5db Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Mon, 18 Dec 2017 11:23:41 +0100 Subject: [PATCH 068/125] Include documentation and examples in source dist --- MANIFEST.in | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/MANIFEST.in b/MANIFEST.in index dd52be9..8c33af0 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,10 @@ +graft docs +graft examples +graft tests +include CHANGELOG.txt +include LICENSE include pymisp/data/*.json include pymisp/data/misp-objects/*.json include pymisp/data/misp-objects/objects/*/definition.json include pymisp/data/misp-objects/relationships/definition.json +include README.md From 6884289156848902a4fc8d6ed6ba02e26450ac22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 18 Dec 2017 13:56:44 +0100 Subject: [PATCH 069/125] chg: Bump describeTypes --- pymisp/data/describeTypes.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pymisp/data/describeTypes.json b/pymisp/data/describeTypes.json index 6349f89..daf0464 100644 --- a/pymisp/data/describeTypes.json +++ b/pymisp/data/describeTypes.json @@ -361,6 +361,10 @@ "default_category": "Attribution", "to_ids": 0 }, + "whois-registrant-org": { + "default_category": "Attribution", + "to_ids": 0 + }, "whois-registrar": { "default_category": "Attribution", "to_ids": 0 @@ -669,6 +673,7 @@ "whois-registrant-email", "whois-registrant-phone", "whois-registrant-name", + "whois-registrant-org", "whois-registrar", "whois-creation-date", "x509-fingerprint-sha1", @@ -989,6 +994,7 @@ "whois-registrant-phone", "whois-registrant-email", "whois-registrant-name", + "whois-registrant-org", "whois-registrar", "whois-creation-date", "comment", From 0fa7785f7e58dae4adc10e1fd5ee394824ee56ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 19 Dec 2017 10:28:49 +0100 Subject: [PATCH 070/125] chg: Bump misp-objects --- pymisp/data/misp-objects | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index b85438f..871b86e 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit b85438fc45b212a21b72d6d2e0df619758fa1444 +Subproject commit 871b86e35fff0c465725e61d34462e0527d668bd From 74640af4ae2f8cc27d40392312646ceeecfbd8b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 19 Dec 2017 17:10:52 +0100 Subject: [PATCH 071/125] fix: Initialize default class parameters. Fix #155 --- pymisp/mispevent.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index c762bf7..1c0a7fc 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -310,6 +310,9 @@ class MISPEvent(AbstractMISP): self._types = describe_types['types'] self.Tag = [] + self.Attribute = [] + self.Object = [] + self.RelatedEvent = [] def _reinitialize_event(self): # Default values for a valid event to send to a MISP instance @@ -445,7 +448,7 @@ class MISPEvent(AbstractMISP): def __repr__(self): if hasattr(self, 'info'): - return '<{self.__class__.__name__}(info={self.info}, date={self.date})'.format(self=self) + return '<{self.__class__.__name__}(info={self.info})'.format(self=self) return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) def from_dict(self, **kwargs): From ef6dd2847814344b046ca8d7ce9bd19a40237dc3 Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Wed, 20 Dec 2017 09:56:55 +0100 Subject: [PATCH 072/125] document submodule downloading --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 4889a54..69b594c 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ pip3 install pymisp ``` git clone https://github.com/MISP/PyMISP.git && cd PyMISP +git submodule update --init pip3 install -I . ``` From e90d28af9d8eae2d7c1c8ce659798595c59eba53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 20 Dec 2017 10:53:46 +0100 Subject: [PATCH 073/125] chg: Add get_attribute_tag method at MISPEvent level Also add a MISPTag class for consistency. --- pymisp/mispevent.py | 58 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 54 insertions(+), 4 deletions(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 1c0a7fc..4c3bdc9 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -129,7 +129,12 @@ class MISPAttribute(AbstractMISP): self.deleted = True def add_tag(self, tag): - self.Tag.append({'name': tag}) + misp_tag = MISPTag() + if isinstance(tag, str): + misp_tag.from_dict(name=tag) + elif isinstance(tag, dict): + misp_tag.from_dict(**tag) + self.Tag.append(misp_tag) def verify(self, gpg_uid): if not has_pyme: @@ -203,7 +208,11 @@ class MISPAttribute(AbstractMISP): if kwargs.get('sharing_group_id'): self.sharing_group_id = int(kwargs.pop('sharing_group_id')) if kwargs.get('Tag'): - self.Tag = [t for t in kwargs.pop('Tag', []) if t] + self.Tag = [] + for tag in kwargs.pop('Tag'): + t = MISPTag() + t.from_dict(**tag) + self.Tag.append(t) # If the user wants to disable correlation, let them. Defaults to False. self.disable_correlation = kwargs.pop("disable_correlation", False) @@ -510,7 +519,11 @@ class MISPEvent(AbstractMISP): sub_event.load(rel_event) self.RelatedEvent.append(sub_event) if kwargs.get('Tag'): - self.Tag = [t for t in kwargs.pop('Tag', []) if t] + self.Tag = [] + for tag in kwargs.pop('Tag'): + t = MISPTag() + t.from_dict(**tag) + self.Tag.append(t) if kwargs.get('Object'): self.Object = [] for obj in kwargs.pop('Object'): @@ -550,9 +563,31 @@ class MISPEvent(AbstractMISP): return to_return def add_tag(self, tag): - self.Tag.append({'name': tag}) + misp_tag = MISPTag() + if isinstance(tag, str): + misp_tag.from_dict(name=tag) + elif isinstance(tag, dict): + misp_tag.from_dict(**tag) + self.Tag.append(misp_tag) + + def get_attribute_tag(self, attribute_identifier): + '''Return the tags associated to an attribute or an object attribute. + :attribute_identifier: can be an ID, UUID, or the value. + ''' + tags = [] + for a in self.attributes + [attribute for o in self.objects for attribute in o.attributes]: + if ((hasattr(a, 'id') and a.id == attribute_identifier) or + (hasattr(a, 'uuid') and a.uuid == attribute_identifier) or + (hasattr(a, 'value') and attribute_identifier == a.value or + attribute_identifier in a.value.split('|'))): + tags += a.tags + return tags def add_attribute_tag(self, tag, attribute_identifier): + '''Add a tag to an existing attribute, raise an Exception if the attribute doesn't exists. + :tag: Tag name + :attribute_identifier: can be an ID, UUID, or the value. + ''' attributes = [] for a in self.attributes: if ((hasattr(a, 'id') and a.id == attribute_identifier) or @@ -626,6 +661,21 @@ class MISPEvent(AbstractMISP): raise InvalidMISPObject("An object to add to an existing Event needs to be either a MISPObject, or a plain python dictionary") +class MISPTag(AbstractMISP): + def __init__(self): + super(MISPTag, self).__init__() + + def __repr__(self): + if hasattr(self, 'name'): + return '<{self.__class__.__name__}(name={self.name})'.format(self=self) + return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) + + def from_dict(self, name, **kwargs): + self.name = name + for k, v in kwargs.items(): + setattr(self, k, v) + + class MISPObjectReference(AbstractMISP): def __init__(self): From 78c156bb6f73cdfccbc0712df9a0b529d7020be2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 20 Dec 2017 12:43:31 +0100 Subject: [PATCH 074/125] new: (hopefully) Cleverer handling of timestamps in the objects & some cleanup --- pymisp/abstract.py | 19 +++++ pymisp/mispevent.py | 175 ++++++++++++++++---------------------------- 2 files changed, 81 insertions(+), 113 deletions(-) diff --git a/pymisp/abstract.py b/pymisp/abstract.py index 0afc0f5..6706804 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -29,6 +29,7 @@ class AbstractMISP(collections.MutableMapping): def __init__(self, **kwargs): super(AbstractMISP, self).__init__() + self.edited = True def properties(self): to_return = [] @@ -43,6 +44,8 @@ class AbstractMISP(collections.MutableMapping): if value is None: continue setattr(self, prop, value) + # We load an existing dictionary, marking it an not-edited + self.edited = False def update_not_jsonable(self, *args): self.__not_jsonable += args @@ -87,3 +90,19 @@ class AbstractMISP(collections.MutableMapping): def __len__(self): return len(self.to_dict()) + + @property + def edited(self): + return self.__edited + + @edited.setter + def edited(self, val): + if isinstance(val, bool): + self.__edited = val + else: + raise Exception('edited can only be True or False') + + def __setattr__(self, name, value): + if name in self.properties(): + self.__edited = True + super(AbstractMISP, self).__setattr__(name, value) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 4c3bdc9..387c5e0 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -2,7 +2,6 @@ # -*- coding: utf-8 -*- import datetime -import time import json import os import base64 @@ -66,6 +65,17 @@ def _int_to_str(d): return d +def _datetime_to_timestamp(d): + if isinstance(d, (int, str)): + # Assume we already have a timestamp + return d + if sys.version_info >= (3, 3): + return d.timestamp() + else: + from datetime import timezone # Only for Python < 3.3 + return (d - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds() + + class MISPAttribute(AbstractMISP): def __init__(self, describe_types=None): @@ -80,31 +90,6 @@ class MISPAttribute(AbstractMISP): self.__sane_default = describe_types['sane_defaults'] self.Tag = [] - def _reinitialize_attribute(self): - # Default values - self.category = None - self.type = None - self.value = None - self.to_ids = False - self.comment = '' - self.distribution = 5 - - # other possible values - self.data = None - self.encrypt = False - self.id = None - self.event_id = None - self.uuid = None - self.timestamp = None - self.sharing_group_id = None - self.deleted = None - self.sig = None - self.SharingGroup = [] - self.ShadowAttribute = [] - self.disable_correlation = False - self.RelatedAttribute = [] - self.Tag = [] - def get_known_types(self): return self._types @@ -219,8 +204,7 @@ class MISPAttribute(AbstractMISP): if self.disable_correlation is None: self.disable_correlation = False - for k, v in kwargs.items(): - setattr(self, k, v) + super(MISPAttribute, self).from_dict(**kwargs) def _prepare_new_malware_sample(self): if '|' in self.value: @@ -284,20 +268,16 @@ class MISPAttribute(AbstractMISP): # DEPRECATED return self.to_dict() - def to_dict(self, with_timestamp=False): - to_return = {} - for attribute in self.properties(): - val = getattr(self, attribute, None) - if val in [None, []]: - continue + def to_dict(self): + to_return = super(MISPAttribute, self).to_dict() + if to_return.get('data'): + to_return['data'] = base64.b64encode(self.data.getvalue()).decode() + + if self.edited: + to_return.pop('timestamp', None) + elif to_return.get('timestamp'): + to_return['timestamp'] = _datetime_to_timestamp(self.timestamp) - if attribute == 'data': - to_return['data'] = base64.b64encode(self.data.getvalue()).decode() - elif attribute == 'timestamp': - if with_timestamp: - to_return['timestamp'] = int(time.mktime(self.timestamp.timetuple())) - else: - to_return[attribute] = val to_return = _int_to_str(to_return) return to_return @@ -323,36 +303,6 @@ class MISPEvent(AbstractMISP): self.Object = [] self.RelatedEvent = [] - def _reinitialize_event(self): - # Default values for a valid event to send to a MISP instance - self.distribution = 3 - self.threat_level_id = 2 - self.analysis = 0 - self.info = None - self.published = False - self.date = datetime.date.today() - - # All other keys - self.sig = None - self.global_sig = None - self.id = None - self.orgc_id = None - self.org_id = None - self.uuid = None - self.attribute_count = None - self.timestamp = None - self.proposal_email_lock = None - self.locked = None - self.publish_timestamp = None - self.sharing_group_id = None - self.Org = None - self.Orgc = None - self.ShadowAttribute = [] - self.RelatedEvent = [] - self.Tag = [] - self.Galaxy = None - self.Object = None - def get_known_types(self): return self._types @@ -531,33 +481,34 @@ class MISPEvent(AbstractMISP): tmp_object.from_dict(**obj) self.Object.append(tmp_object) - for k, v in kwargs.items(): - setattr(self, k, v) + super(MISPEvent, self).from_dict(**kwargs) def _json(self): # DEPTECATED return self.to_dict() - def to_dict(self, with_timestamp=False): + def to_dict(self): + for o in self.objects: + if o.edited: + self.edited = True + break + for a in self.attributes: + if a.edited: + self.edited = True + break + to_return = super(MISPEvent, self).to_dict() + if to_return.get('date'): to_return['date'] = self.date.isoformat() - if with_timestamp and to_return.get('timestamp'): - if sys.version_info >= (3, 3): - to_return['timestamp'] = self.timestamp.timestamp() - else: - from datetime import timezone # Only for Python < 3.3 - to_return['timestamp'] = (self.timestamp - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds() - else: + if self.edited: to_return.pop('timestamp', None) - if with_timestamp and to_return.get('publish_timestamp'): - if sys.version_info >= (3, 3): - to_return['publish_timestamp'] = self.publish_timestamp.timestamp() - else: - from datetime import timezone # Only for Python < 3.3 - to_return['publish_timestamp'] = (self.publish_timestamp - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds() - else: - to_return.pop('publish_timestamp', None) + elif to_return.get('timestamp'): + to_return['timestamp'] = _datetime_to_timestamp(self.timestamp) + + if to_return.get('publish_timestamp'): + to_return['publish_timestamp'] = _datetime_to_timestamp(self.publish_timestamp) + to_return = _int_to_str(to_return) to_return = {'Event': to_return} return to_return @@ -672,8 +623,7 @@ class MISPTag(AbstractMISP): def from_dict(self, name, **kwargs): self.name = name - for k, v in kwargs.items(): - setattr(self, k, v) + super(MISPTag, self).from_dict(**kwargs) class MISPObjectReference(AbstractMISP): @@ -691,8 +641,7 @@ class MISPObjectReference(AbstractMISP): self.referenced_uuid = referenced_uuid self.relationship_type = relationship_type self.comment = comment - for k, v in kwargs.items(): - setattr(self, k, v) + super(MISPObjectReference, self).from_dict(**kwargs) class MISPUser(AbstractMISP): @@ -700,20 +649,12 @@ class MISPUser(AbstractMISP): def __init__(self): super(MISPUser, self).__init__() - def from_dict(self, **kwargs): - for k, v in kwargs.items(): - setattr(self, k, v) - class MISPOrganisation(AbstractMISP): def __init__(self): super(MISPOrganisation, self).__init__() - def from_dict(self, **kwargs): - for k, v in kwargs.items(): - setattr(self, k, v) - class MISPObjectAttribute(MISPAttribute): @@ -824,22 +765,30 @@ class MISPObject(AbstractMISP): else: self.__known_template = False - for key, value in kwargs.items(): - if key == 'Attribute': - for v in value: - self.add_attribute(**v) - elif key == 'ObjectReference': - for v in value: - self.add_reference(**v) - else: - setattr(self, key, value) + if kwargs.get('Attribute'): + for a in kwargs.pop('Attribute'): + self.add_attribute(**a) + if kwargs.get('ObjectReference'): + for r in kwargs.pop('ObjectReference'): + self.add_reference(**r) + + super(MISPObject, self).from_dict(**kwargs) def to_dict(self, strict=False): - # Set the expected key (Attributes) - self.Attribute = self.attributes + for a in self.attributes: + if a.edited: + self.edited = True + break if strict or self.__strict and self.__known_template: self._validate() - return super(MISPObject, self).to_dict() + # Set the expected key (Attributes) + self.Attribute = self.attributes + to_return = super(MISPObject, self).to_dict() + if self.edited: + to_return.pop('timestamp', None) + elif to_return.get('timestamp'): + to_return['timestamp'] = _datetime_to_timestamp(self.timestamp) + return to_return def to_json(self, strict=False): if strict or self.__strict and self.__known_template: From 582fd28702fa3923c2f3c0790fd2a5e2fa683b81 Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Wed, 20 Dec 2017 13:29:05 +0100 Subject: [PATCH 075/125] fix MISPObject missing distribution and sharing_group_id - fix MISPObject missing distribution concept - fix language typo paramaters => parameters --- pymisp/mispevent.py | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 1c0a7fc..6f4558a 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -699,7 +699,7 @@ class MISPObjectAttribute(MISPAttribute): class MISPObject(AbstractMISP): - def __init__(self, name, strict=False, standalone=False, default_attributes_paramaters={}, **kwargs): + def __init__(self, name, strict=False, standalone=False, default_attributes_parameters={}, **kwargs): ''' Master class representing a generic MISP object :name: Name of the object @@ -708,7 +708,7 @@ class MISPObject(AbstractMISP): :standalone: The object will be pushed as directly on MISP, not as a part of an event. In this case the ObjectReference needs to be pushed manually and cannot be in the JSON dump. - :default_attributes_paramaters: Used as template for the attributes if they are not overwritten in add_attribute + :default_attributes_parameters: Used as template for the attributes if they are not overwritten in add_attribute ''' super(MISPObject, self).__init__(**kwargs) self.__strict = strict @@ -735,21 +735,25 @@ class MISPObject(AbstractMISP): pass self.uuid = str(uuid.uuid4()) self.__fast_attribute_access = {} # Hashtable object_relation: [attributes] - self._default_attributes_paramaters = default_attributes_paramaters - if self._default_attributes_paramaters: + self._default_attributes_parameters = default_attributes_parameters + if self._default_attributes_parameters: # Let's clean that up - self._default_attributes_paramaters.pop('value', None) # duh - self._default_attributes_paramaters.pop('uuid', None) # duh - self._default_attributes_paramaters.pop('id', None) # duh - self._default_attributes_paramaters.pop('object_id', None) # duh - self._default_attributes_paramaters.pop('type', None) # depends on the value - self._default_attributes_paramaters.pop('object_relation', None) # depends on the value - self._default_attributes_paramaters.pop('disable_correlation', None) # depends on the value - self._default_attributes_paramaters.pop('to_ids', None) # depends on the value - self._default_attributes_paramaters.pop('category', None) # depends on the value - self._default_attributes_paramaters.pop('deleted', None) # doesn't make sense to pre-set it - self._default_attributes_paramaters.pop('data', None) # in case the original in a sample or an attachment - self.distribution = self._default_attributes_paramaters.distribution + self._default_attributes_parameters.pop('value', None) # duh + self._default_attributes_parameters.pop('uuid', None) # duh + self._default_attributes_parameters.pop('id', None) # duh + self._default_attributes_parameters.pop('object_id', None) # duh + self._default_attributes_parameters.pop('type', None) # depends on the value + self._default_attributes_parameters.pop('object_relation', None) # depends on the value + self._default_attributes_parameters.pop('disable_correlation', None) # depends on the value + self._default_attributes_parameters.pop('to_ids', None) # depends on the value + self._default_attributes_parameters.pop('category', None) # depends on the value + self._default_attributes_parameters.pop('deleted', None) # doesn't make sense to pre-set it + self._default_attributes_parameters.pop('data', None) # in case the original in a sample or an attachment + self.distribution = self._default_attributes_parameters.distribution + self.sharing_group_id = self._default_attributes_parameters.sharing_group_id + else: + self.distribution = 3 + self.sharing_group_id = None self.ObjectReference = [] self._standalone = standalone if self._standalone: @@ -858,8 +862,8 @@ class MISPObject(AbstractMISP): attribute = MISPObjectAttribute({}) else: attribute = MISPObjectAttribute({}) - # Overwrite the parameters of self._default_attributes_paramaters with the ones of value - attribute.from_dict(object_relation=object_relation, **dict(self._default_attributes_paramaters, **value)) + # Overwrite the parameters of self._default_attributes_parameters with the ones of value + attribute.from_dict(object_relation=object_relation, **dict(self._default_attributes_parameters, **value)) if not self.__fast_attribute_access.get(object_relation): self.__fast_attribute_access[object_relation] = [] self.__fast_attribute_access[object_relation].append(attribute) From 994afea0bd7c198aa1a7540e89293bf422ecfc72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 20 Dec 2017 14:21:32 +0100 Subject: [PATCH 076/125] fix: Fix python2.7 support. --- pymisp/mispevent.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 387c5e0..3279cbe 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -72,8 +72,7 @@ def _datetime_to_timestamp(d): if sys.version_info >= (3, 3): return d.timestamp() else: - from datetime import timezone # Only for Python < 3.3 - return (d - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds() + return (d - datetime.utcfromtimestamp(0)).total_seconds() class MISPAttribute(AbstractMISP): From 4a1d43c7e24a1ee6a00df59a323345b4fd40baef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 20 Dec 2017 14:27:31 +0100 Subject: [PATCH 077/125] fix: Fix typo --- pymisp/tools/create_misp_object.py | 22 +++++++++++----------- pymisp/tools/elfobject.py | 2 +- pymisp/tools/machoobject.py | 2 +- pymisp/tools/peobject.py | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/pymisp/tools/create_misp_object.py b/pymisp/tools/create_misp_object.py index e12f76e..cc0c6fc 100644 --- a/pymisp/tools/create_misp_object.py +++ b/pymisp/tools/create_misp_object.py @@ -22,8 +22,8 @@ class FileTypeNotImplemented(MISPObjectException): pass -def make_pe_objects(lief_parsed, misp_file, standalone=True, default_attributes_paramaters={}): - pe_object = PEObject(parsed=lief_parsed, standalone=standalone, default_attributes_paramaters=default_attributes_paramaters) +def make_pe_objects(lief_parsed, misp_file, standalone=True, default_attributes_parameters={}): + pe_object = PEObject(parsed=lief_parsed, standalone=standalone, default_attributes_parameters=default_attributes_parameters) misp_file.add_reference(pe_object.uuid, 'included-in', 'PE indicators') pe_sections = [] for s in pe_object.sections: @@ -31,8 +31,8 @@ def make_pe_objects(lief_parsed, misp_file, standalone=True, default_attributes_ return misp_file, pe_object, pe_sections -def make_elf_objects(lief_parsed, misp_file, standalone=True, default_attributes_paramaters={}): - elf_object = ELFObject(parsed=lief_parsed, standalone=standalone, default_attributes_paramaters=default_attributes_paramaters) +def make_elf_objects(lief_parsed, misp_file, standalone=True, default_attributes_parameters={}): + elf_object = ELFObject(parsed=lief_parsed, standalone=standalone, default_attributes_parameters=default_attributes_parameters) misp_file.add_reference(elf_object.uuid, 'included-in', 'ELF indicators') elf_sections = [] for s in elf_object.sections: @@ -40,8 +40,8 @@ def make_elf_objects(lief_parsed, misp_file, standalone=True, default_attributes return misp_file, elf_object, elf_sections -def make_macho_objects(lief_parsed, misp_file, standalone=True, default_attributes_paramaters={}): - macho_object = MachOObject(parsed=lief_parsed, standalone=standalone, default_attributes_paramaters=default_attributes_paramaters) +def make_macho_objects(lief_parsed, misp_file, standalone=True, default_attributes_parameters={}): + macho_object = MachOObject(parsed=lief_parsed, standalone=standalone, default_attributes_parameters=default_attributes_parameters) misp_file.add_reference(macho_object.uuid, 'included-in', 'MachO indicators') macho_sections = [] for s in macho_object.sections: @@ -49,9 +49,9 @@ def make_macho_objects(lief_parsed, misp_file, standalone=True, default_attribut return misp_file, macho_object, macho_sections -def make_binary_objects(filepath=None, pseudofile=None, filename=None, standalone=True, default_attributes_paramaters={}): +def make_binary_objects(filepath=None, pseudofile=None, filename=None, standalone=True, default_attributes_parameters={}): misp_file = FileObject(filepath=filepath, pseudofile=pseudofile, filename=filename, - standalone=standalone, default_attributes_paramaters=default_attributes_paramaters) + standalone=standalone, default_attributes_parameters=default_attributes_parameters) if HAS_LIEF and filepath or (pseudofile and filename): try: if filepath: @@ -63,11 +63,11 @@ def make_binary_objects(filepath=None, pseudofile=None, filename=None, standalon else: lief_parsed = lief.parse(raw=pseudofile.getvalue(), name=filename) if isinstance(lief_parsed, lief.PE.Binary): - return make_pe_objects(lief_parsed, misp_file, standalone, default_attributes_paramaters) + return make_pe_objects(lief_parsed, misp_file, standalone, default_attributes_parameters) elif isinstance(lief_parsed, lief.ELF.Binary): - return make_elf_objects(lief_parsed, misp_file, standalone, default_attributes_paramaters) + return make_elf_objects(lief_parsed, misp_file, standalone, default_attributes_parameters) elif isinstance(lief_parsed, lief.MachO.Binary): - return make_macho_objects(lief_parsed, misp_file, standalone, default_attributes_paramaters) + return make_macho_objects(lief_parsed, misp_file, standalone, default_attributes_parameters) except lief.bad_format as e: logger.warning('Bad format: {}'.format(e)) except lief.bad_file as e: diff --git a/pymisp/tools/elfobject.py b/pymisp/tools/elfobject.py index 4dda680..d58390c 100644 --- a/pymisp/tools/elfobject.py +++ b/pymisp/tools/elfobject.py @@ -58,7 +58,7 @@ class ELFObject(AbstractMISPObjectGenerator): if self.__elf.sections: pos = 0 for section in self.__elf.sections: - s = ELFSectionObject(section, self._standalone, default_attributes_paramaters=self._default_attributes_paramaters) + s = ELFSectionObject(section, self._standalone, default_attributes_parameters=self._default_attributes_parameters) self.add_reference(s.uuid, 'included-in', 'Section {} of ELF'.format(pos)) pos += 1 self.sections.append(s) diff --git a/pymisp/tools/machoobject.py b/pymisp/tools/machoobject.py index fddda21..ed6e2ae 100644 --- a/pymisp/tools/machoobject.py +++ b/pymisp/tools/machoobject.py @@ -61,7 +61,7 @@ class MachOObject(AbstractMISPObjectGenerator): if self.__macho.sections: pos = 0 for section in self.__macho.sections: - s = MachOSectionObject(section, self._standalone, default_attributes_paramaters=self._default_attributes_paramaters) + s = MachOSectionObject(section, self._standalone, default_attributes_parameters=self._default_attributes_parameters) self.add_reference(s.uuid, 'included-in', 'Section {} of MachO'.format(pos)) pos += 1 self.sections.append(s) diff --git a/pymisp/tools/peobject.py b/pymisp/tools/peobject.py index 8f32426..d55a97b 100644 --- a/pymisp/tools/peobject.py +++ b/pymisp/tools/peobject.py @@ -104,7 +104,7 @@ class PEObject(AbstractMISPObjectGenerator): if self.__pe.sections: pos = 0 for section in self.__pe.sections: - s = PESectionObject(section, self._standalone, default_attributes_paramaters=self._default_attributes_paramaters) + s = PESectionObject(section, self._standalone, default_attributes_parameters=self._default_attributes_parameters) self.add_reference(s.uuid, 'included-in', 'Section {} of PE'.format(pos)) if ((self.__pe.entrypoint >= section.virtual_address) and (self.__pe.entrypoint < (section.virtual_address + section.virtual_size))): From f3b8029bb37790c7c8bba3c5ec47ab3879423d0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 20 Dec 2017 14:33:44 +0100 Subject: [PATCH 078/125] fix: properly call datetime.datetime.utcfromtimestamp --- pymisp/mispevent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index c683efb..71dece1 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -72,7 +72,7 @@ def _datetime_to_timestamp(d): if sys.version_info >= (3, 3): return d.timestamp() else: - return (d - datetime.utcfromtimestamp(0)).total_seconds() + return (d - datetime.datetime.utcfromtimestamp(0)).total_seconds() class MISPAttribute(AbstractMISP): From 9c4e98f025dde02151923670eb87322db907696d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 20 Dec 2017 16:59:52 +0100 Subject: [PATCH 079/125] fix: Forgotten calls to master class. --- pymisp/mispevent.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 71dece1..be330ce 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -78,6 +78,7 @@ def _datetime_to_timestamp(d): class MISPAttribute(AbstractMISP): def __init__(self, describe_types=None): + super(MISPAttribute, self).__init__() if not describe_types: ressources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') with open(os.path.join(ressources_path, 'describeTypes.json'), 'r') as f: @@ -284,6 +285,7 @@ class MISPAttribute(AbstractMISP): class MISPEvent(AbstractMISP): def __init__(self, describe_types=None, strict_validation=False): + super(MISPEvent, self).__init__() ressources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') if strict_validation: with open(os.path.join(ressources_path, 'schema.json'), 'r') as f: From 749acd70b5ff00f2f74809b51cfb6db66b9543df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 21 Dec 2017 09:38:41 +0100 Subject: [PATCH 080/125] fix: Add setter for Attribute in MISPEvent --- pymisp/mispevent.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index be330ce..1977820 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -584,6 +584,10 @@ class MISPEvent(AbstractMISP): def attributes(self): return self.Attribute + @attributes.setter + def attributes(self, attributes): + self.Attribute = attributes + @property def related_events(self): return self.RelatedEvent From b745fce44681ef647c3c43fe21bd4f7a14125906 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 21 Dec 2017 09:46:09 +0100 Subject: [PATCH 081/125] chg: Validate attributes in attributes.setter --- pymisp/mispevent.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 1977820..5d16cd1 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -586,7 +586,10 @@ class MISPEvent(AbstractMISP): @attributes.setter def attributes(self, attributes): - self.Attribute = attributes + if all(isinstance(x, MISPAttribute) for x in attributes): + self.Attribute = attributes + else: + raise PyMISPError('All the attributes have to be of type MISPAttribute.') @property def related_events(self): From ae644bdbf58de3d7985b117224c71c1185872d97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 21 Dec 2017 16:27:40 +0100 Subject: [PATCH 082/125] chg: Bump describeTypes.json --- pymisp/data/describeTypes.json | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pymisp/data/describeTypes.json b/pymisp/data/describeTypes.json index daf0464..ca3bea0 100644 --- a/pymisp/data/describeTypes.json +++ b/pymisp/data/describeTypes.json @@ -121,6 +121,10 @@ "default_category": "Payload installation", "to_ids": 1 }, + "stix2-pattern": { + "default_category": "Payload installation", + "to_ids": 1 + }, "sigma": { "default_category": "Payload installation", "to_ids": 1 @@ -613,6 +617,7 @@ "pattern-in-traffic", "pattern-in-memory", "yara", + "stix2-pattern", "sigma", "cookie", "vulnerability", @@ -820,6 +825,7 @@ "AS", "pattern-in-file", "pattern-in-traffic", + "stix2-pattern", "yara", "sigma", "attachment", @@ -879,6 +885,7 @@ "pattern-in-file", "pattern-in-memory", "pdb", + "stix2-pattern", "yara", "sigma", "attachment", @@ -930,6 +937,7 @@ "pattern-in-file", "pattern-in-traffic", "pattern-in-memory", + "stix2-pattern", "yara", "sigma", "vulnerability", @@ -973,6 +981,7 @@ "AS", "snort", "pattern-in-file", + "stix2-pattern", "pattern-in-traffic", "attachment", "comment", From 9e9bad731d7247557796f4e317db4508c23f4e7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 21 Dec 2017 17:25:12 +0100 Subject: [PATCH 083/125] new: Add last field to get_csv --- pymisp/api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pymisp/api.py b/pymisp/api.py index a4d91fc..9bbb7ae 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1606,7 +1606,7 @@ class PyMISP(object): def get_stix(self, **kwargs): return self.get_stix_event(**kwargs) - def get_csv(self, eventid=None, attributes=[], object_attributes=[], misp_types=[], context=False, ignore=False): + def get_csv(self, eventid=None, attributes=[], object_attributes=[], misp_types=[], context=False, ignore=False, last=None): """Get MISP values in CSV format :param eventid: The event ID to query :param attributes: The column names to export from normal attributes (i.e. uuid, value, type, ...) @@ -1632,6 +1632,8 @@ class PyMISP(object): to_post['includeContext'] = True if ignore: to_post['ignore'] = True + if last: + to_post['last'] = last if to_post: response = self.__prepare_request('POST', url, json.dumps(to_post), output_type='json') else: From efb6ca974c284445fd91cf57c8949781800b0e96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 21 Dec 2017 18:46:28 +0100 Subject: [PATCH 084/125] fix: Properly use the edited flag --- pymisp/abstract.py | 39 +++++- pymisp/mispevent.py | 317 ++++++++++++++++++++++---------------------- 2 files changed, 194 insertions(+), 162 deletions(-) diff --git a/pymisp/abstract.py b/pymisp/abstract.py index 6706804..6578940 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -2,6 +2,8 @@ # -*- coding: utf-8 -*- import abc +import sys +import datetime import json from json import JSONEncoder import collections @@ -29,8 +31,9 @@ class AbstractMISP(collections.MutableMapping): def __init__(self, **kwargs): super(AbstractMISP, self).__init__() - self.edited = True + self.__edited = True + @property def properties(self): to_return = [] for prop, value in vars(self).items(): @@ -45,7 +48,7 @@ class AbstractMISP(collections.MutableMapping): continue setattr(self, prop, value) # We load an existing dictionary, marking it an not-edited - self.edited = False + self.__edited = False def update_not_jsonable(self, *args): self.__not_jsonable += args @@ -59,10 +62,19 @@ class AbstractMISP(collections.MutableMapping): def to_dict(self): to_return = {} - for attribute in self.properties(): + for attribute in self.properties: val = getattr(self, attribute, None) if val is None: continue + if attribute == 'timestamp': + if self.edited: + # In order to be accepted by MISP, the timestamp of an object + # needs to be either newer, or None. + # If the current object is marked as edited, the easiest is to + # skip the timestamp and let MISP deal with it + continue + else: + val = self._datetime_to_timestamp(val) to_return[attribute] = val return to_return @@ -93,6 +105,16 @@ class AbstractMISP(collections.MutableMapping): @property def edited(self): + if self.__edited: + return self.__edited + for p in self.properties: + if self.__edited: + break + if isinstance(p, AbstractMISP) and p.edited: + self.__edited = True + elif isinstance(p, list) and all(isinstance(a, AbstractMISP) for a in p): + if any(a.edited for a in p): + self.__edited = True return self.__edited @edited.setter @@ -103,6 +125,15 @@ class AbstractMISP(collections.MutableMapping): raise Exception('edited can only be True or False') def __setattr__(self, name, value): - if name in self.properties(): + if name in self.properties: self.__edited = True super(AbstractMISP, self).__setattr__(name, value) + + def _datetime_to_timestamp(self, d): + if isinstance(d, (int, str)): + # Assume we already have a timestamp + return d + if sys.version_info >= (3, 3): + return d.timestamp() + else: + return (d - datetime.datetime.utcfromtimestamp(0)).total_seconds() diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 5d16cd1..9b68771 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -65,16 +65,6 @@ def _int_to_str(d): return d -def _datetime_to_timestamp(d): - if isinstance(d, (int, str)): - # Assume we already have a timestamp - return d - if sys.version_info >= (3, 3): - return d.timestamp() - else: - return (d - datetime.datetime.utcfromtimestamp(0)).total_seconds() - - class MISPAttribute(AbstractMISP): def __init__(self, describe_types=None): @@ -90,25 +80,26 @@ class MISPAttribute(AbstractMISP): self.__sane_default = describe_types['sane_defaults'] self.Tag = [] - def get_known_types(self): + @property + def known_types(self): return self._types - def _serialize(self): - return '{type}{category}{to_ids}{uuid}{timestamp}{comment}{deleted}{value}'.format( - type=self.type, category=self.category, to_ids=self.to_ids, uuid=self.uuid, timestamp=self.timestamp, - comment=self.comment, deleted=self.deleted, value=self.value).encode() + @property + def malware_binary(self): + if hasattr(self, '_malware_binary'): + return self._malware_binary + return None - def sign(self, gpg_uid, passphrase=None): - if not has_pyme: - raise PyMISPError('pyme is required, please install: pip install --pre pyme3. You will also need libgpg-error-dev and libgpgme11-dev.') - to_sign = self._serialize() - with gpg.Context() as c: - keys = list(c.keylist(gpg_uid)) - c.signers = keys[:1] - if passphrase: - c.set_passphrase_cb(lambda *args: passphrase) - signed, _ = c.sign(to_sign, mode=mode.DETACH) - self.sig = base64.b64encode(signed).decode() + @property + def tags(self): + return self.Tag + + @tags.setter + def tags(self, tags): + if all(isinstance(x, MISPTag) for x in tags): + self.Tag = tags + else: + raise PyMISPError('All the attributes have to be of type MISPAttribute.') def delete(self): self.deleted = True @@ -119,23 +110,10 @@ class MISPAttribute(AbstractMISP): misp_tag.from_dict(name=tag) elif isinstance(tag, dict): misp_tag.from_dict(**tag) - self.Tag.append(misp_tag) - - def verify(self, gpg_uid): - if not has_pyme: - raise PyMISPError('pyme is required, please install: pip install --pre pyme3. You will also need libgpg-error-dev and libgpgme11-dev.') - signed_data = self._serialize() - with gpg.Context() as c: - keys = list(c.keylist(gpg_uid)) - try: - c.verify(signed_data, signature=base64.b64decode(self.sig), verify=keys[:1]) - return {self.uuid: True} - except Exception: - return {self.uuid: False} - - def set_all_values(self, **kwargs): - # to be deprecated - self.from_dict(**kwargs) + else: + raise PyMISPError("The tag is in an invalid format (can be either string, or list): {}".format(tag)) + self.tags.append(misp_tag) + self.edited = True def __repr__(self): if hasattr(self, 'value'): @@ -147,8 +125,8 @@ class MISPAttribute(AbstractMISP): if kwargs['type'] not in self.__category_type_mapping[kwargs['category']]: raise NewAttributeError('{} and {} is an invalid combination, type for this category has to be in {}'.format( kwargs.get('type'), kwargs.get('category'), (', '.join(self.__category_type_mapping[kwargs['category']])))) - # Required - self.type = kwargs.pop('type', None) + + self.type = kwargs.pop('type', None) # Required if self.type is None: raise NewAttributeError('The type of the attribute is required.') if self.type not in self.get_known_types(): @@ -255,31 +233,64 @@ class MISPAttribute(AbstractMISP): # not a encrypted zip file, assuming it is a new malware sample self._prepare_new_malware_sample() + def to_dict(self): + to_return = super(MISPAttribute, self).to_dict() + if to_return.get('data'): + to_return['data'] = base64.b64encode(self.data.getvalue()).decode() + to_return = _int_to_str(to_return) + return to_return + + def verify(self, gpg_uid): + # Not used + if not has_pyme: + raise PyMISPError('pyme is required, please install: pip install --pre pyme3. You will also need libgpg-error-dev and libgpgme11-dev.') + signed_data = self._serialize() + with gpg.Context() as c: + keys = list(c.keylist(gpg_uid)) + try: + c.verify(signed_data, signature=base64.b64decode(self.sig), verify=keys[:1]) + return {self.uuid: True} + except Exception: + return {self.uuid: False} + + def _serialize(self): + # Not used + return '{type}{category}{to_ids}{uuid}{timestamp}{comment}{deleted}{value}'.format( + type=self.type, category=self.category, to_ids=self.to_ids, uuid=self.uuid, timestamp=self.timestamp, + comment=self.comment, deleted=self.deleted, value=self.value).encode() + + def sign(self, gpg_uid, passphrase=None): + # Not used + if not has_pyme: + raise PyMISPError('pyme is required, please install: pip install --pre pyme3. You will also need libgpg-error-dev and libgpgme11-dev.') + to_sign = self._serialize() + with gpg.Context() as c: + keys = list(c.keylist(gpg_uid)) + c.signers = keys[:1] + if passphrase: + c.set_passphrase_cb(lambda *args: passphrase) + signed, _ = c.sign(to_sign, mode=mode.DETACH) + self.sig = base64.b64encode(signed).decode() + + def get_known_types(self): + # Deprecated + return self.known_types + def get_malware_binary(self): - if hasattr(self, '_malware_binary'): - return self._malware_binary - return None + # Deprecated + return self.malware_binary def _json(self): # DEPRECATED return self.to_dict() def _json_full(self): - # DEPRECATED + # Deprecated return self.to_dict() - def to_dict(self): - to_return = super(MISPAttribute, self).to_dict() - if to_return.get('data'): - to_return['data'] = base64.b64encode(self.data.getvalue()).decode() - - if self.edited: - to_return.pop('timestamp', None) - elif to_return.get('timestamp'): - to_return['timestamp'] = _datetime_to_timestamp(self.timestamp) - - to_return = _int_to_str(to_return) - return to_return + def set_all_values(self, **kwargs): + # Deprecated + self.from_dict(**kwargs) class MISPEvent(AbstractMISP): @@ -304,66 +315,10 @@ class MISPEvent(AbstractMISP): self.Object = [] self.RelatedEvent = [] - def get_known_types(self): + @property + def known_types(self): return self._types - def _serialize(self): - return '{date}{threat_level_id}{info}{uuid}{analysis}{timestamp}'.format( - date=self.date, threat_level_id=self.threat_level_id, info=self.info, - uuid=self.uuid, analysis=self.analysis, timestamp=self.timestamp).encode() - - def _serialize_sigs(self): - all_sigs = self.sig - for a in self.attributes: - all_sigs += a.sig - return all_sigs.encode() - - def sign(self, gpg_uid, passphrase=None): - if not has_pyme: - raise PyMISPError('pyme is required, please install: pip install --pre pyme3. You will also need libgpg-error-dev and libgpgme11-dev.') - to_sign = self._serialize() - with gpg.Context() as c: - keys = list(c.keylist(gpg_uid)) - c.signers = keys[:1] - if passphrase: - c.set_passphrase_cb(lambda *args: passphrase) - signed, _ = c.sign(to_sign, mode=mode.DETACH) - self.sig = base64.b64encode(signed).decode() - for a in self.attributes: - a.sign(gpg_uid, passphrase) - to_sign_global = self._serialize_sigs() - with gpg.Context() as c: - keys = list(c.keylist(gpg_uid)) - c.signers = keys[:1] - if passphrase: - c.set_passphrase_cb(lambda *args: passphrase) - signed, _ = c.sign(to_sign_global, mode=mode.DETACH) - self.global_sig = base64.b64encode(signed).decode() - - def verify(self, gpg_uid): - if not has_pyme: - raise PyMISPError('pyme is required, please install: pip install --pre pyme3. You will also need libgpg-error-dev and libgpgme11-dev.') - to_return = {} - signed_data = self._serialize() - with gpg.Context() as c: - keys = list(c.keylist(gpg_uid)) - try: - c.verify(signed_data, signature=base64.b64decode(self.sig), verify=keys[:1]) - to_return[self.uuid] = True - except Exception: - to_return[self.uuid] = False - for a in self.attributes: - to_return.update(a.verify(gpg_uid)) - to_verify_global = self._serialize_sigs() - with gpg.Context() as c: - keys = list(c.keylist(gpg_uid)) - try: - c.verify(to_verify_global, signature=base64.b64decode(self.global_sig), verify=keys[:1]) - to_return['global'] = True - except Exception: - to_return['global'] = False - return to_return - def load_file(self, event_path): if not os.path.exists(event_path): raise PyMISPError('Invalid path, unable to load the event.') @@ -402,10 +357,6 @@ class MISPEvent(AbstractMISP): else: raise NewEventError('Invalid format for the date: {} - {}'.format(date, type(date))) - def set_all_values(self, **kwargs): - # to be deprecated - self.from_dict(**kwargs) - def __repr__(self): if hasattr(self, 'info'): return '<{self.__class__.__name__}(info={self.info})'.format(self=self) @@ -484,31 +435,13 @@ class MISPEvent(AbstractMISP): super(MISPEvent, self).from_dict(**kwargs) - def _json(self): - # DEPTECATED - return self.to_dict() - def to_dict(self): - for o in self.objects: - if o.edited: - self.edited = True - break - for a in self.attributes: - if a.edited: - self.edited = True - break - to_return = super(MISPEvent, self).to_dict() if to_return.get('date'): to_return['date'] = self.date.isoformat() - if self.edited: - to_return.pop('timestamp', None) - elif to_return.get('timestamp'): - to_return['timestamp'] = _datetime_to_timestamp(self.timestamp) - if to_return.get('publish_timestamp'): - to_return['publish_timestamp'] = _datetime_to_timestamp(self.publish_timestamp) + to_return['publish_timestamp'] = self._datetime_to_timestamp(self.publish_timestamp) to_return = _int_to_str(to_return) to_return = {'Event': to_return} @@ -520,7 +453,10 @@ class MISPEvent(AbstractMISP): misp_tag.from_dict(name=tag) elif isinstance(tag, dict): misp_tag.from_dict(**tag) - self.Tag.append(misp_tag) + else: + raise PyMISPError("The tag is in an invalid format (can be either string, or list): {}".format(tag)) + self.tags.append(misp_tag) + self.edited = True def get_attribute_tag(self, attribute_identifier): '''Return the tags associated to an attribute or an object attribute. @@ -550,6 +486,7 @@ class MISPEvent(AbstractMISP): attributes.append(a) if not attributes: raise Exception('No attribute with identifier {} found.'.format(attribute_identifier)) + self.edited = True return attributes def publish(self): @@ -570,15 +507,16 @@ class MISPEvent(AbstractMISP): raise Exception('No attribute with UUID/ID {} found.'.format(attribute_id)) def add_attribute(self, type, value, **kwargs): - attribute = MISPAttribute() if isinstance(value, list): for a in value: - self.add_attribute(type, a, **kwargs) + self.add_attribute(type=type, value=a, **kwargs) else: + attribute = MISPAttribute() attribute.set_all_values(type=type, value=value, **kwargs) if not hasattr(self, 'attributes'): self.attributes = [] self.attributes.append(attribute) + self.edited = True @property def attributes(self): @@ -618,6 +556,76 @@ class MISPEvent(AbstractMISP): self.Object.append(tmp_object) else: raise InvalidMISPObject("An object to add to an existing Event needs to be either a MISPObject, or a plain python dictionary") + self.edited = True + + def _serialize(self): + return '{date}{threat_level_id}{info}{uuid}{analysis}{timestamp}'.format( + date=self.date, threat_level_id=self.threat_level_id, info=self.info, + uuid=self.uuid, analysis=self.analysis, timestamp=self.timestamp).encode() + + def _serialize_sigs(self): + all_sigs = self.sig + for a in self.attributes: + all_sigs += a.sig + return all_sigs.encode() + + def sign(self, gpg_uid, passphrase=None): + if not has_pyme: + raise PyMISPError('pyme is required, please install: pip install --pre pyme3. You will also need libgpg-error-dev and libgpgme11-dev.') + to_sign = self._serialize() + with gpg.Context() as c: + keys = list(c.keylist(gpg_uid)) + c.signers = keys[:1] + if passphrase: + c.set_passphrase_cb(lambda *args: passphrase) + signed, _ = c.sign(to_sign, mode=mode.DETACH) + self.sig = base64.b64encode(signed).decode() + for a in self.attributes: + a.sign(gpg_uid, passphrase) + to_sign_global = self._serialize_sigs() + with gpg.Context() as c: + keys = list(c.keylist(gpg_uid)) + c.signers = keys[:1] + if passphrase: + c.set_passphrase_cb(lambda *args: passphrase) + signed, _ = c.sign(to_sign_global, mode=mode.DETACH) + self.global_sig = base64.b64encode(signed).decode() + + def verify(self, gpg_uid): + if not has_pyme: + raise PyMISPError('pyme is required, please install: pip install --pre pyme3. You will also need libgpg-error-dev and libgpgme11-dev.') + to_return = {} + signed_data = self._serialize() + with gpg.Context() as c: + keys = list(c.keylist(gpg_uid)) + try: + c.verify(signed_data, signature=base64.b64decode(self.sig), verify=keys[:1]) + to_return[self.uuid] = True + except Exception: + to_return[self.uuid] = False + for a in self.attributes: + to_return.update(a.verify(gpg_uid)) + to_verify_global = self._serialize_sigs() + with gpg.Context() as c: + keys = list(c.keylist(gpg_uid)) + try: + c.verify(to_verify_global, signature=base64.b64decode(self.global_sig), verify=keys[:1]) + to_return['global'] = True + except Exception: + to_return['global'] = False + return to_return + + def get_known_types(self): + # Deprecated + return self.known_types + + def set_all_values(self, **kwargs): + # Deprecated + self.from_dict(**kwargs) + + def _json(self): + # DEPTECATED + return self.to_dict() class MISPTag(AbstractMISP): @@ -734,6 +742,7 @@ class MISPObject(AbstractMISP): pass self.uuid = str(uuid.uuid4()) self.__fast_attribute_access = {} # Hashtable object_relation: [attributes] + self.Attribute = [] self._default_attributes_parameters = default_attributes_parameters if self._default_attributes_parameters: # Let's clean that up @@ -787,20 +796,9 @@ class MISPObject(AbstractMISP): super(MISPObject, self).from_dict(**kwargs) def to_dict(self, strict=False): - for a in self.attributes: - if a.edited: - self.edited = True - break if strict or self.__strict and self.__known_template: self._validate() - # Set the expected key (Attributes) - self.Attribute = self.attributes - to_return = super(MISPObject, self).to_dict() - if self.edited: - to_return.pop('timestamp', None) - elif to_return.get('timestamp'): - to_return['timestamp'] = _datetime_to_timestamp(self.timestamp) - return to_return + return super(MISPObject, self).to_dict() def to_json(self, strict=False): if strict or self.__strict and self.__known_template: @@ -840,6 +838,7 @@ class MISPObject(AbstractMISP): reference.from_dict(object_uuid=object_uuid, referenced_uuid=referenced_uuid, relationship_type=relationship_type, comment=comment, **kwargs) self.ObjectReference.append(reference) + self.edited = True def get_attributes_by_relation(self, object_relation): '''Returns the list of attributes with the given object relation in the object''' @@ -851,7 +850,7 @@ class MISPObject(AbstractMISP): @property def attributes(self): - return [a for sublist in self.__fast_attribute_access.values() for a in sublist] + return self.Attribute @property def references(self): @@ -873,5 +872,7 @@ class MISPObject(AbstractMISP): attribute.from_dict(object_relation=object_relation, **dict(self._default_attributes_parameters, **value)) if not self.__fast_attribute_access.get(object_relation): self.__fast_attribute_access[object_relation] = [] + self.Attribute.append(attribute) self.__fast_attribute_access[object_relation].append(attribute) + self.edited = True return attribute From a497613a85c841b5934174b7f0de57f9893a2dc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 22 Dec 2017 14:49:14 +0100 Subject: [PATCH 085/125] chg: Update documentation, cleanup --- pymisp/__init__.py | 21 ++++ pymisp/abstract.py | 22 +++- pymisp/api.py | 29 +---- pymisp/mispevent.py | 259 ++++++++++++++++++++++++-------------------- 4 files changed, 186 insertions(+), 145 deletions(-) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 5de8bee..e9ad297 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,10 +1,31 @@ __version__ = '2.4.84' import sys import logging +import functools +import warnings + logger = logging.getLogger(__name__) FORMAT = "%(levelname)s [%(filename)s:%(lineno)s - %(funcName)s() ] %(message)s" logging.basicConfig(stream=sys.stderr, level=logging.WARNING, format=FORMAT) + +def deprecated(func): + '''This is a decorator which can be used to mark functions + as deprecated. It will result in a warning being emitted + when the function is used.''' + + @functools.wraps(func) + def new_func(*args, **kwargs): + warnings.showwarning( + "Call to deprecated function {}.".format(func.__name__), + category=DeprecationWarning, + filename=func.__code__.co_filename, + lineno=func.__code__.co_firstlineno + 1 + ) + return func(*args, **kwargs) + return new_func + + try: from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate # noqa from .api import PyMISP # noqa diff --git a/pymisp/abstract.py b/pymisp/abstract.py index 6578940..de0c40d 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -30,11 +30,15 @@ class AbstractMISP(collections.MutableMapping): __not_jsonable = [] def __init__(self, **kwargs): + """Abstract class for all the MISP objects""" super(AbstractMISP, self).__init__() - self.__edited = True + self.__edited = True # As we create a new object, we assume it is edited @property def properties(self): + """All the class public properties that will be dumped in the dictionary, and the JSON export. + Note: all the properties starting with a `_` (private), or listed in __not_jsonable will be skipped. + """ to_return = [] for prop, value in vars(self).items(): if prop.startswith('_') or prop in self.__not_jsonable: @@ -43,6 +47,11 @@ class AbstractMISP(collections.MutableMapping): return to_return def from_dict(self, **kwargs): + """Loading all the parameters as class properties, if they aren't `None`. + This method aims to be called when all the properties requiring a special + treatment are processed. + Note: This method is used when you initialize an object with existing data so by default, + the class is flaged as not edited.""" for prop, value in kwargs.items(): if value is None: continue @@ -51,9 +60,11 @@ class AbstractMISP(collections.MutableMapping): self.__edited = False def update_not_jsonable(self, *args): + """Add entries to the __not_jsonable list""" self.__not_jsonable += args def set_not_jsonable(self, *args): + """Set __not_jsonable to a new list""" self.__not_jsonable = args def from_json(self, json_string): @@ -61,6 +72,9 @@ class AbstractMISP(collections.MutableMapping): self.from_dict(json.loads(json_string)) def to_dict(self): + """Dump the lass to a dictionary. + This method automatically removes the timestamp recursively in every object + that has been edited is order to let MISP update the event accordingly.""" to_return = {} for attribute in self.properties: val = getattr(self, attribute, None) @@ -79,9 +93,11 @@ class AbstractMISP(collections.MutableMapping): return to_return def jsonable(self): + """This method is used by the JSON encoder""" return self.to_dict() def to_json(self): + """Dump recursively any class of type MISPAbstract to a json string""" return json.dumps(self, cls=MISPEncode) def __getitem__(self, key): @@ -105,6 +121,8 @@ class AbstractMISP(collections.MutableMapping): @property def edited(self): + """Recursively check if an object has been edited and update the flag accordingly + to the parent objects""" if self.__edited: return self.__edited for p in self.properties: @@ -119,6 +137,7 @@ class AbstractMISP(collections.MutableMapping): @edited.setter def edited(self, val): + """Set the edit flag""" if isinstance(val, bool): self.__edited = val else: @@ -130,6 +149,7 @@ class AbstractMISP(collections.MutableMapping): super(AbstractMISP, self).__setattr__(name, value) def _datetime_to_timestamp(self, d): + """Convert a datetime.datetime object to a timestamp (int)""" if isinstance(d, (int, str)): # Assume we already have a timestamp return d diff --git a/pymisp/api.py b/pymisp/api.py index 9bbb7ae..1d25d38 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -10,13 +10,11 @@ from dateutil.parser import parse import os import base64 import re -import functools import logging -import warnings from io import BytesIO, open import zipfile -from . import __version__ +from . import __version__, deprecated from .exceptions import PyMISPError, SearchError, NoURL, NoKey from .mispevent import MISPEvent, MISPAttribute, MISPUser, MISPOrganisation from .abstract import MISPEncode @@ -46,23 +44,6 @@ except ImportError: ASYNC_OK = False -def deprecated(func): - '''This is a decorator which can be used to mark functions - as deprecated. It will result in a warning being emitted - when the function is used.''' - - @functools.wraps(func) - def new_func(*args, **kwargs): - warnings.showwarning( - "Call to deprecated function {}.".format(func.__name__), - category=DeprecationWarning, - filename=func.__code__.co_filename, - lineno=func.__code__.co_firstlineno + 1 - ) - return func(*args, **kwargs) - return new_func - - class PyMISP(object): """Python API for MISP @@ -282,8 +263,8 @@ class PyMISP(object): def _prepare_full_event(self, distribution, threat_level_id, analysis, info, date=None, published=False, orgc_id=None, org_id=None, sharing_group_id=None): """Initialize a new MISPEvent from scratch""" misp_event = MISPEvent(self.describe_types) - misp_event.set_all_values(info=info, distribution=distribution, threat_level_id=threat_level_id, - analysis=analysis, date=date, orgc_id=orgc_id, org_id=org_id, sharing_group_id=sharing_group_id) + misp_event.from_dict(info=info, distribution=distribution, threat_level_id=threat_level_id, + analysis=analysis, date=date, orgc_id=orgc_id, org_id=org_id, sharing_group_id=sharing_group_id) if published: misp_event.publish() return misp_event @@ -291,8 +272,8 @@ class PyMISP(object): def _prepare_full_attribute(self, category, type_value, value, to_ids, comment=None, distribution=None, **kwargs): """Initialize a new MISPAttribute from scratch""" misp_attribute = MISPAttribute(self.describe_types) - misp_attribute.set_all_values(type=type_value, value=value, category=category, - to_ids=to_ids, comment=comment, distribution=distribution, **kwargs) + misp_attribute.from_dict(type=type_value, value=value, category=category, + to_ids=to_ids, comment=comment, distribution=distribution, **kwargs) return misp_attribute def _valid_uuid(self, uuid): diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 9b68771..55fc57c 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -12,6 +12,7 @@ import sys import uuid from collections import Counter +from . import deprecated from .abstract import AbstractMISP from .exceptions import UnknownMISPObjectTemplate, InvalidMISPObject, PyMISPError, NewEventError, NewAttributeError @@ -102,9 +103,11 @@ class MISPAttribute(AbstractMISP): raise PyMISPError('All the attributes have to be of type MISPAttribute.') def delete(self): + """Mark the attribute as deleted (soft delete)""" self.deleted = True def add_tag(self, tag): + """Add a tag to the attribute (by name or a MISPTag object)""" misp_tag = MISPTag() if isinstance(tag, str): misp_tag.from_dict(name=tag) @@ -115,11 +118,6 @@ class MISPAttribute(AbstractMISP): self.tags.append(misp_tag) self.edited = True - def __repr__(self): - if hasattr(self, 'value'): - return '<{self.__class__.__name__}(type={self.type}, value={self.value})'.format(self=self) - return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) - def from_dict(self, **kwargs): if kwargs.get('type') and kwargs.get('category'): if kwargs['type'] not in self.__category_type_mapping[kwargs['category']]: @@ -129,7 +127,7 @@ class MISPAttribute(AbstractMISP): self.type = kwargs.pop('type', None) # Required if self.type is None: raise NewAttributeError('The type of the attribute is required.') - if self.type not in self.get_known_types(): + if self.type not in self.known_types: raise NewAttributeError('{} is invalid, type has to be in {}'.format(self.type, (', '.join(self._types)))) type_defaults = self.__sane_default[self.type] @@ -184,6 +182,13 @@ class MISPAttribute(AbstractMISP): super(MISPAttribute, self).from_dict(**kwargs) + def to_dict(self): + to_return = super(MISPAttribute, self).to_dict() + if to_return.get('data'): + to_return['data'] = base64.b64encode(self.data.getvalue()).decode() + to_return = _int_to_str(to_return) + return to_return + def _prepare_new_malware_sample(self): if '|' in self.value: # Get the filename, ignore the md5, because humans. @@ -233,12 +238,10 @@ class MISPAttribute(AbstractMISP): # not a encrypted zip file, assuming it is a new malware sample self._prepare_new_malware_sample() - def to_dict(self): - to_return = super(MISPAttribute, self).to_dict() - if to_return.get('data'): - to_return['data'] = base64.b64encode(self.data.getvalue()).decode() - to_return = _int_to_str(to_return) - return to_return + def __repr__(self): + if hasattr(self, 'value'): + return '<{self.__class__.__name__}(type={self.type}, value={self.value})'.format(self=self) + return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) def verify(self, gpg_uid): # Not used @@ -272,24 +275,24 @@ class MISPAttribute(AbstractMISP): signed, _ = c.sign(to_sign, mode=mode.DETACH) self.sig = base64.b64encode(signed).decode() + @deprecated def get_known_types(self): - # Deprecated return self.known_types + @deprecated def get_malware_binary(self): - # Deprecated return self.malware_binary + @deprecated def _json(self): - # DEPRECATED return self.to_dict() + @deprecated def _json_full(self): - # Deprecated return self.to_dict() + @deprecated def set_all_values(self, **kwargs): - # Deprecated self.from_dict(**kwargs) @@ -319,13 +322,38 @@ class MISPEvent(AbstractMISP): def known_types(self): return self._types + @property + def attributes(self): + return self.Attribute + + @attributes.setter + def attributes(self, attributes): + if all(isinstance(x, MISPAttribute) for x in attributes): + self.Attribute = attributes + else: + raise PyMISPError('All the attributes have to be of type MISPAttribute.') + + @property + def related_events(self): + return self.RelatedEvent + + @property + def objects(self): + return self.Object + + @property + def tags(self): + return self.Tag + def load_file(self, event_path): + """Load a JSON dump from a file on the disk""" if not os.path.exists(event_path): raise PyMISPError('Invalid path, unable to load the event.') with open(event_path, 'r') as f: self.load(f) def load(self, json_event): + """Load a JSON dump from a pseudo file or a JSON string""" if hasattr(json_event, 'read'): # python2 and python3 compatible to find if we have a file json_event = json_event.read() @@ -342,9 +370,10 @@ class MISPEvent(AbstractMISP): event['Event']['attribute_count'] = '0' jsonschema.validate(event, self.__json_schema) e = event.get('Event') - self.set_all_values(**e) + self.from_dict(**e) def set_date(self, date, ignore_invalid=False): + """Set a date for the event (string, datetime, or date object)""" if isinstance(date, basestring) or isinstance(date, unicode): self.date = parse(date).date() elif isinstance(date, datetime.datetime): @@ -357,11 +386,6 @@ class MISPEvent(AbstractMISP): else: raise NewEventError('Invalid format for the date: {} - {}'.format(date, type(date))) - def __repr__(self): - if hasattr(self, 'info'): - return '<{self.__class__.__name__}(info={self.info})'.format(self=self) - return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) - def from_dict(self, **kwargs): # Required value self.info = kwargs.pop('info', None) @@ -396,7 +420,7 @@ class MISPEvent(AbstractMISP): if kwargs.get('Attribute'): for a in kwargs.pop('Attribute'): attribute = MISPAttribute() - attribute.set_all_values(**a) + attribute.from_dict(**a) if not hasattr(self, 'Attribute'): self.Attribute = [] self.Attribute.append(attribute) @@ -448,6 +472,7 @@ class MISPEvent(AbstractMISP): return to_return def add_tag(self, tag): + """Add a tag to the attribute (by name or a MISPTag object)""" misp_tag = MISPTag() if isinstance(tag, str): misp_tag.from_dict(name=tag) @@ -490,12 +515,15 @@ class MISPEvent(AbstractMISP): return attributes def publish(self): + """Mark the attribute as published""" self.published = True def unpublish(self): + """Mark the attribute as un-published (set publish flag to false)""" self.published = False def delete_attribute(self, attribute_id): + """Delete an attribute, you can search by ID or UUID""" found = False for a in self.attributes: if ((hasattr(a, 'id') and a.id == attribute_id) or @@ -507,47 +535,28 @@ class MISPEvent(AbstractMISP): raise Exception('No attribute with UUID/ID {} found.'.format(attribute_id)) def add_attribute(self, type, value, **kwargs): + """Add an attribute. type and value are required but you can pass all + other parameters supported by MISPAttribute""" if isinstance(value, list): for a in value: self.add_attribute(type=type, value=a, **kwargs) else: attribute = MISPAttribute() - attribute.set_all_values(type=type, value=value, **kwargs) + attribute.from_dict(type=type, value=value, **kwargs) if not hasattr(self, 'attributes'): self.attributes = [] self.attributes.append(attribute) self.edited = True - @property - def attributes(self): - return self.Attribute - - @attributes.setter - def attributes(self, attributes): - if all(isinstance(x, MISPAttribute) for x in attributes): - self.Attribute = attributes - else: - raise PyMISPError('All the attributes have to be of type MISPAttribute.') - - @property - def related_events(self): - return self.RelatedEvent - - @property - def objects(self): - return self.Object - - @property - def tags(self): - return self.Tag - def get_object_by_id(self, object_id): + """Get an object by ID (the ID is the one set by the server when creating the new object)""" for obj in self.objects: if hasattr(obj, 'id') and obj.id == object_id: return obj raise InvalidMISPObject('Object with {} does not exists in ths event'.format(object_id)) def add_object(self, obj): + """Add an object to the Event, either by passing a MISPObject, or a dictionary""" if isinstance(obj, MISPObject): self.Object.append(obj) elif isinstance(obj, dict): @@ -558,18 +567,25 @@ class MISPEvent(AbstractMISP): raise InvalidMISPObject("An object to add to an existing Event needs to be either a MISPObject, or a plain python dictionary") self.edited = True + def __repr__(self): + if hasattr(self, 'info'): + return '<{self.__class__.__name__}(info={self.info})'.format(self=self) + return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) + def _serialize(self): return '{date}{threat_level_id}{info}{uuid}{analysis}{timestamp}'.format( date=self.date, threat_level_id=self.threat_level_id, info=self.info, uuid=self.uuid, analysis=self.analysis, timestamp=self.timestamp).encode() def _serialize_sigs(self): + # Not used all_sigs = self.sig for a in self.attributes: all_sigs += a.sig return all_sigs.encode() def sign(self, gpg_uid, passphrase=None): + # Not used if not has_pyme: raise PyMISPError('pyme is required, please install: pip install --pre pyme3. You will also need libgpg-error-dev and libgpgme11-dev.') to_sign = self._serialize() @@ -592,6 +608,7 @@ class MISPEvent(AbstractMISP): self.global_sig = base64.b64encode(signed).decode() def verify(self, gpg_uid): + # Not used if not has_pyme: raise PyMISPError('pyme is required, please install: pip install --pre pyme3. You will also need libgpg-error-dev and libgpgme11-dev.') to_return = {} @@ -615,16 +632,16 @@ class MISPEvent(AbstractMISP): to_return['global'] = False return to_return + @deprecated def get_known_types(self): - # Deprecated return self.known_types + @deprecated def set_all_values(self, **kwargs): - # Deprecated self.from_dict(**kwargs) + @deprecated def _json(self): - # DEPTECATED return self.to_dict() @@ -632,26 +649,21 @@ class MISPTag(AbstractMISP): def __init__(self): super(MISPTag, self).__init__() + def from_dict(self, name, **kwargs): + self.name = name + super(MISPTag, self).from_dict(**kwargs) + def __repr__(self): if hasattr(self, 'name'): return '<{self.__class__.__name__}(name={self.name})'.format(self=self) return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) - def from_dict(self, name, **kwargs): - self.name = name - super(MISPTag, self).from_dict(**kwargs) - class MISPObjectReference(AbstractMISP): def __init__(self): super(MISPObjectReference, self).__init__() - def __repr__(self): - if hasattr(self, 'referenced_uuid'): - return '<{self.__class__.__name__}(object_uuid={self.object_uuid}, referenced_uuid={self.referenced_uuid}, relationship_type={self.relationship_type})'.format(self=self) - return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) - def from_dict(self, object_uuid, referenced_uuid, relationship_type, comment=None, **kwargs): self.object_uuid = object_uuid self.referenced_uuid = referenced_uuid @@ -659,6 +671,11 @@ class MISPObjectReference(AbstractMISP): self.comment = comment super(MISPObjectReference, self).from_dict(**kwargs) + def __repr__(self): + if hasattr(self, 'referenced_uuid'): + return '<{self.__class__.__name__}(object_uuid={self.object_uuid}, referenced_uuid={self.referenced_uuid}, relationship_type={self.relationship_type})'.format(self=self) + return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) + class MISPUser(AbstractMISP): @@ -678,11 +695,6 @@ class MISPObjectAttribute(MISPAttribute): super(MISPObjectAttribute, self).__init__() self.__definition = definition - def __repr__(self): - if hasattr(self, 'value'): - return '<{self.__class__.__name__}(object_relation={self.object_relation}, value={self.value})'.format(self=self) - return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) - def from_dict(self, object_relation, value, **kwargs): self.object_relation = object_relation self.value = value @@ -703,6 +715,11 @@ class MISPObjectAttribute(MISPAttribute): kwargs.update(**self) super(MISPObjectAttribute, self).from_dict(**kwargs) + def __repr__(self): + if hasattr(self, 'value'): + return '<{self.__class__.__name__}(object_relation={self.object_relation}, value={self.value})'.format(self=self) + return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) + class MISPObject(AbstractMISP): @@ -760,7 +777,7 @@ class MISPObject(AbstractMISP): self.distribution = self._default_attributes_parameters.distribution self.sharing_group_id = self._default_attributes_parameters.sharing_group_id else: - self.distribution = 3 + self.distribution = 5 # Default to inherit self.sharing_group_id = None self.ObjectReference = [] self._standalone = standalone @@ -768,10 +785,13 @@ class MISPObject(AbstractMISP): # Mark as non_jsonable because we need to add the references manually after the object(s) have been created self.update_not_jsonable('ObjectReference') - def __repr__(self): - if hasattr(self, 'name'): - return '<{self.__class__.__name__}(name={self.name})'.format(self=self) - return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) + @property + def attributes(self): + return self.Attribute + + @property + def references(self): + return self.ObjectReference def from_dict(self, **kwargs): if self.__known_template: @@ -795,6 +815,51 @@ class MISPObject(AbstractMISP): super(MISPObject, self).from_dict(**kwargs) + def add_reference(self, referenced_uuid, relationship_type, comment=None, **kwargs): + """Add a link (uuid) to an other object""" + if kwargs.get('object_uuid'): + # Load existing object + object_uuid = kwargs.pop('object_uuid') + else: + # New reference + object_uuid = self.uuid + reference = MISPObjectReference() + reference.from_dict(object_uuid=object_uuid, referenced_uuid=referenced_uuid, + relationship_type=relationship_type, comment=comment, **kwargs) + self.ObjectReference.append(reference) + self.edited = True + + def get_attributes_by_relation(self, object_relation): + '''Returns the list of attributes with the given object relation in the object''' + return self.__fast_attribute_access.get(object_relation, []) + + def has_attributes_by_relation(self, list_of_relations): + '''True if all the relations in the list are defined in the object''' + return all(relation in self.__fast_attribute_access for relation in list_of_relations) + + def add_attribute(self, object_relation, **value): + """Add an attribute. object_relation is required and the value key is a + dictionary with all the keys supported by MISPAttribute""" + if value.get('value') is None: + return None + if self.__known_template: + if self.__definition['attributes'].get(object_relation): + attribute = MISPObjectAttribute(self.__definition['attributes'][object_relation]) + else: + # Woopsie, this object_relation is unknown, no sane defaults for you. + logger.warning("The template ({}) doesn't have the object_relation ({}) you're trying to add.".format(self.name, object_relation)) + attribute = MISPObjectAttribute({}) + else: + attribute = MISPObjectAttribute({}) + # Overwrite the parameters of self._default_attributes_parameters with the ones of value + attribute.from_dict(object_relation=object_relation, **dict(self._default_attributes_parameters, **value)) + if not self.__fast_attribute_access.get(object_relation): + self.__fast_attribute_access[object_relation] = [] + self.Attribute.append(attribute) + self.__fast_attribute_access[object_relation].append(attribute) + self.edited = True + return attribute + def to_dict(self, strict=False): if strict or self.__strict and self.__known_template: self._validate() @@ -826,53 +891,7 @@ class MISPObject(AbstractMISP): raise InvalidMISPObject('{} is required'.format(r)) return True - def add_reference(self, referenced_uuid, relationship_type, comment=None, **kwargs): - """Add a link (uuid) to an other object""" - if kwargs.get('object_uuid'): - # Load existing object - object_uuid = kwargs.pop('object_uuid') - else: - # New reference - object_uuid = self.uuid - reference = MISPObjectReference() - reference.from_dict(object_uuid=object_uuid, referenced_uuid=referenced_uuid, - relationship_type=relationship_type, comment=comment, **kwargs) - self.ObjectReference.append(reference) - self.edited = True - - def get_attributes_by_relation(self, object_relation): - '''Returns the list of attributes with the given object relation in the object''' - return self.__fast_attribute_access.get(object_relation, []) - - def has_attributes_by_relation(self, list_of_relations): - '''True if all the relations in the list are defined in the object''' - return all(relation in self.__fast_attribute_access for relation in list_of_relations) - - @property - def attributes(self): - return self.Attribute - - @property - def references(self): - return self.ObjectReference - - def add_attribute(self, object_relation, **value): - if value.get('value') is None: - return None - if self.__known_template: - if self.__definition['attributes'].get(object_relation): - attribute = MISPObjectAttribute(self.__definition['attributes'][object_relation]) - else: - # Woopsie, this object_relation is unknown, no sane defaults for you. - logger.warning("The template ({}) doesn't have the object_relation ({}) you're trying to add.".format(self.name, object_relation)) - attribute = MISPObjectAttribute({}) - else: - attribute = MISPObjectAttribute({}) - # Overwrite the parameters of self._default_attributes_parameters with the ones of value - attribute.from_dict(object_relation=object_relation, **dict(self._default_attributes_parameters, **value)) - if not self.__fast_attribute_access.get(object_relation): - self.__fast_attribute_access[object_relation] = [] - self.Attribute.append(attribute) - self.__fast_attribute_access[object_relation].append(attribute) - self.edited = True - return attribute + def __repr__(self): + if hasattr(self, 'name'): + return '<{self.__class__.__name__}(name={self.name})'.format(self=self) + return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) From e8e13f3218804ff1016785db830111913f182a03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 22 Dec 2017 17:22:07 +0100 Subject: [PATCH 086/125] chg: Update documentation --- docs/source/conf.py | 4 --- docs/source/index.rst | 6 ++-- docs/source/modules.rst | 80 ++++++++++++++++++++++++++++++++++++++++- docs/source/pymisp.rst | 22 ------------ docs/source/tools.rst | 69 +++++++++++++++++++++++++++++++++++ pymisp/__init__.py | 2 +- pymisp/api.py | 5 +-- pymisp/tools/openioc.py | 14 -------- 8 files changed, 152 insertions(+), 50 deletions(-) delete mode 100644 docs/source/pymisp.rst create mode 100644 docs/source/tools.rst diff --git a/docs/source/conf.py b/docs/source/conf.py index bdaadc8..78715f6 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -17,10 +17,6 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # -import os -import sys -sys.path.insert(0, os.path.abspath('.')) - from recommonmark.parser import CommonMarkParser # -- General configuration ------------------------------------------------ diff --git a/docs/source/index.rst b/docs/source/index.rst index 68863a5..f519501 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -9,12 +9,11 @@ Welcome to PyMISP's documentation! Contents: .. toctree:: - :maxdepth: 2 + :maxdepth: 4 README modules - - + tools Indices and tables ================== @@ -22,4 +21,3 @@ Indices and tables * :ref:`genindex` * :ref:`modindex` * :ref:`search` - diff --git a/docs/source/modules.rst b/docs/source/modules.rst index 1bb98dd..7db7414 100644 --- a/docs/source/modules.rst +++ b/docs/source/modules.rst @@ -4,4 +4,82 @@ pymisp .. toctree:: :maxdepth: 4 - pymisp +.. automodule:: pymisp + :members: + + +PyMISP +------ + +.. autoclass:: PyMISP + :members: + +MISPAbstract +------------ + +.. autoclass:: AbstractMISP + :members: + +MISPEncode +---------- + +.. autoclass:: MISPEncode + :members: + +MISPEvent +--------- + +.. autoclass:: MISPEvent + :members: + :inherited-members: + +MISPAttribute +------------- + +.. autoclass:: MISPAttribute + :members: + :inherited-members: + +MISPObject +---------- + +.. autoclass:: MISPObject + :members: + :inherited-members: + +MISPObjectAttribute +------------------- + +.. autoclass:: MISPObjectAttribute + :members: + :inherited-members: + +MISPObjectReference +------------------- + +.. autoclass:: MISPObjectReference + :members: + :inherited-members: + +MISPTag +------- + +.. autoclass:: MISPTag + :members: + :inherited-members: + +MISPUser +-------- + +.. autoclass:: MISPUser + :members: + :inherited-members: + + +MISPOrganisation +---------------- + +.. autoclass:: MISPOrganisation + :members: + :inherited-members: + diff --git a/docs/source/pymisp.rst b/docs/source/pymisp.rst deleted file mode 100644 index 28ca0d9..0000000 --- a/docs/source/pymisp.rst +++ /dev/null @@ -1,22 +0,0 @@ -pymisp package -============== - -Submodules ----------- - -pymisp.api module ------------------ - -.. automodule:: pymisp.api - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: pymisp - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/tools.rst b/docs/source/tools.rst new file mode 100644 index 0000000..36e1e8f --- /dev/null +++ b/docs/source/tools.rst @@ -0,0 +1,69 @@ +pymisp - Tools +============== + +.. toctree:: + :maxdepth: 4 + +.. automodule:: pymisp.tools + :members: + +File Object +----------- + +.. autoclass:: FileObject + :members: + :inherited-members: + +ELF Object +---------- + +.. autoclass:: ELFObject + :members: + :inherited-members: + +.. autoclass:: ELFSectionObject + :members: + :inherited-members: + +PE Object +--------- + +.. autoclass:: PEObject + :members: + :inherited-members: + +.. autoclass:: PESectionObject + :members: + :inherited-members: + +Mach-O Object +------------- + +.. autoclass:: MachOObject + :members: + :inherited-members: + +.. autoclass:: MachOSectionObject + :members: + :inherited-members: + +VT Report Object +---------------- + +.. autoclass:: VTReportObject + :members: + :inherited-members: + +STIX +---- + +.. automodule:: pymisp.tools.stix + :members: + +OpenIOC +-------- + +.. automethod:: pymisp.tools.load_openioc + +.. automethod:: pymisp.tools.load_openioc_file + diff --git a/pymisp/__init__.py b/pymisp/__init__.py index e9ad297..5624525 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -30,7 +30,7 @@ try: from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate # noqa from .api import PyMISP # noqa from .abstract import AbstractMISP, MISPEncode # noqa - from .mispevent import MISPEvent, MISPAttribute, MISPObjectReference, MISPObjectAttribute, MISPObject # noqa + from .mispevent import MISPEvent, MISPAttribute, MISPObjectReference, MISPObjectAttribute, MISPObject, MISPTag, MISPUser, MISPOrganisation # noqa from .tools import AbstractMISPObjectGenerator # noqa from .tools import Neo4j # noqa from .tools import stix # noqa diff --git a/pymisp/api.py b/pymisp/api.py index 1d25d38..b885b16 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -49,10 +49,7 @@ class PyMISP(object): :param url: URL of the MISP instance you want to connect to :param key: API key of the user you want to use - :param ssl: can be True or False (to check ot not the validity - of the certificate. Or a CA_BUNDLE in case of self - signed certiifcate (the concatenation of all the - *.crt of the chain) + :param ssl: can be True or False (to check ot not the validity of the certificate. Or a CA_BUNDLE in case of self signed certiifcate (the concatenation of all the \*.crt of the chain) :param out_type: Type of object (json) NOTE: XML output isn't supported anymore, keeping the flag for compatibility reasons. :param debug: Write all the debug information to stderr :param proxies: Proxy dict as describes here: http://docs.python-requests.org/en/master/user/advanced/#proxies diff --git a/pymisp/tools/openioc.py b/pymisp/tools/openioc.py index ea0c880..1c2329a 100755 --- a/pymisp/tools/openioc.py +++ b/pymisp/tools/openioc.py @@ -278,17 +278,3 @@ def set_all_attributes(openioc, misp_event): misp_event.add_attribute(**attribute_values) return misp_event - - -if __name__ == '__main__': - import requests - # test file for composite - url = 'https://raw.githubusercontent.com/fireeye/iocs/master/BlogPosts/9cee306d-5441-4cd3-932d-f3119752634c.ioc' - # ~ url = 'https://raw.githubusercontent.com/MISP/misp-modules/master/tests/openioc.xml' - x = requests.get(url) - mispEvent = load_openioc(x.text) - print(mispEvent) - # ~ from pymisp import PyMISP - # ~ misp = PyMISP('http://misp.local', 'xxxxx') - # ~ r = misp.add_event(mispEvent) - # ~ print(r) From a02e777a99ee49775213fc295ad1e219cc726b7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 22 Dec 2017 17:37:54 +0100 Subject: [PATCH 087/125] chg: Version bump --- pymisp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 5624525..64c7b1b 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.4.84' +__version__ = '2.4.85' import sys import logging import functools From adfecc797588b57713158cbb1476a3db8dbcaf7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 22 Dec 2017 17:39:01 +0100 Subject: [PATCH 088/125] chg: Bump misp-objects --- pymisp/data/misp-objects | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index 871b86e..1460d05 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit 871b86e35fff0c465725e61d34462e0527d668bd +Subproject commit 1460d055a0207668cf1f0e99ff347411038f0113 From f447f55268b89cc0c00b0a3fecf47b335c1c7efa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 22 Dec 2017 17:41:26 +0100 Subject: [PATCH 089/125] chg: Bump Changelog. --- CHANGELOG.txt | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index bad9051..f18c409 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -2,6 +2,71 @@ Changelog ========= +v2.4.85 (2017-12-22) +-------------------- + +New +~~~ +- Add last field to get_csv. [Raphaël Vinot] +- (hopefully) Cleverer handling of timestamps in the objects. [Raphaël + Vinot] + + & some cleanup + +Changes +~~~~~~~ +- Bump misp-objects. [Raphaël Vinot] +- Version bump. [Raphaël Vinot] +- Update documentation. [Raphaël Vinot] +- Update documentation, cleanup. [Raphaël Vinot] +- Bump describeTypes.json. [Raphaël Vinot] +- Validate attributes in attributes.setter. [Raphaël Vinot] +- Add get_attribute_tag method at MISPEvent level. [Raphaël Vinot] + + Also add a MISPTag class for consistency. +- Bump misp-objects. [Raphaël Vinot] +- Bump describeTypes. [Raphaël Vinot] +- Add __repr__ methods (fix last commit) [Raphaël Vinot] +- Add __repr__ methods. [Raphaël Vinot] +- Use new format for filtering. [Raphaël Vinot] +- Bump misp-objects. [Raphaël Vinot] +- Bump describeTypes. [Raphaël Vinot] + +Fix +~~~ +- Properly use the edited flag. [Raphaël Vinot] +- Add setter for Attribute in MISPEvent. [Raphaël Vinot] +- Forgotten calls to master class. [Raphaël Vinot] +- Properly call datetime.datetime.utcfromtimestamp. [Raphaël Vinot] +- Fix typo. [Raphaël Vinot] +- Fix python2.7 support. [Raphaël Vinot] +- Initialize default class parameters. [Raphaël Vinot] + + Fix #155 + +Other +~~~~~ +- Merge branch 'cvandeplas-master' [Raphaël Vinot] +- Merge branch 'master' of https://github.com/cvandeplas/PyMISP into + cvandeplas-master. [Raphaël Vinot] +- Merge remote-tracking branch 'MISP/master' [Christophe Vandeplas] +- Fix MISPObject missing distribution and sharing_group_id. [Christophe + Vandeplas] + + - fix MISPObject missing distribution concept + - fix language typo paramaters => parameters +- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot] +- Merge pull request #156 from cvandeplas/master. [Alexandre Dulaunoy] + + document submodule downloading +- Document submodule downloading. [Christophe Vandeplas] +- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot] +- Merge pull request #154 from wagner-certat/inc-meta. [Raphaël Vinot] + + Include documentation and examples in source dist +- Include documentation and examples in source dist. [Sebastian Wagner] + + v2.4.84 (2017-12-13) -------------------- @@ -15,6 +80,8 @@ New Changes ~~~~~~~ +- Bump misp-objects. [Raphaël Vinot] +- Bump Changelog. [Raphaël Vinot] - Bump version. [Raphaël Vinot] - Make the library easier to use. [Raphaël Vinot] - Allow to pass a pseudofile to LIEF. [Raphaël Vinot] From 4708786959c2459ab996b52bfa67ca0bf14b1668 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 26 Dec 2017 17:13:57 +0100 Subject: [PATCH 090/125] chg: Add MISPSighting class. --- pymisp/__init__.py | 2 +- pymisp/api.py | 26 ++++++++++++++++++++++---- pymisp/mispevent.py | 25 +++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 64c7b1b..a0e8c5f 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -30,7 +30,7 @@ try: from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate # noqa from .api import PyMISP # noqa from .abstract import AbstractMISP, MISPEncode # noqa - from .mispevent import MISPEvent, MISPAttribute, MISPObjectReference, MISPObjectAttribute, MISPObject, MISPTag, MISPUser, MISPOrganisation # noqa + from .mispevent import MISPEvent, MISPAttribute, MISPObjectReference, MISPObjectAttribute, MISPObject, MISPTag, MISPUser, MISPOrganisation, MISPSighting # noqa from .tools import AbstractMISPObjectGenerator # noqa from .tools import Neo4j # noqa from .tools import stix # noqa diff --git a/pymisp/api.py b/pymisp/api.py index b885b16..75e84b0 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -16,7 +16,7 @@ import zipfile from . import __version__, deprecated from .exceptions import PyMISPError, SearchError, NoURL, NoKey -from .mispevent import MISPEvent, MISPAttribute, MISPUser, MISPOrganisation +from .mispevent import MISPEvent, MISPAttribute, MISPUser, MISPOrganisation, MISPSighting from .abstract import MISPEncode logger = logging.getLogger('pymisp') @@ -1296,10 +1296,16 @@ class PyMISP(object): def set_sightings(self, sightings): """Push a sighting (python dictionary)""" - if isinstance(sightings, dict): - sightings = json.dumps(sightings) + to_post = [] + if not isinstance(sightings, list): + sightings = [sightings] + for sighting in sightings: + if isinstance(sighting, MISPSighting): + to_post.appent(sighting.to_json()) + elif isinstance(sighting, dict): + to_post.append(json.dumps(sightings)) url = urljoin(self.root_url, 'sightings/add/') - response = self.__prepare_request('POST', url, sightings) + response = self.__prepare_request('POST', url, json.dumps(to_post)) return self._check_response(response) def sighting_per_json(self, json_file): @@ -1308,6 +1314,18 @@ class PyMISP(object): jdata = json.load(f) return self.set_sightings(jdata) + def sighting(self, value, source=None, type=None, timestamp=None, **kwargs): + """ Set a single sighting. + :value: Value can either be the attribute's value (to update sighting on all the attributes with this value), + or an UUID in order to update the sightings of one particular attribute. + :source: Source of the sighting + :type: Type of the sighting + :timestamp: Timestamp associated to the sighting + """ + s = MISPSighting() + s.from_dict(value=value, source=source, type=type, timestamp=timestamp, **kwargs) + return self.set_sightings(s) + # ############## Sharing Groups ################## def get_sharing_groups(self): diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 55fc57c..abec80f 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -689,6 +689,31 @@ class MISPOrganisation(AbstractMISP): super(MISPOrganisation, self).__init__() +class MISPSighting(AbstractMISP): + + def __init__(self): + super(MISPSighting, self).__init__() + + def from_dict(self, value, source=None, type=None, timestamp=None, **kwargs): + """Initialize the MISPSighting from a dictionary + :value: Value can either be the attribute's value (to update sighting on all the attributes with this value), + or an UUID in order to update the sightings of one particular attribute. + :source: Source of the sighting + :type: Type of the sighting + :timestamp: Timestamp associated to the sighting + """ + self.value = value + self.source = source + self.type = type + self.timestamp = timestamp + super(MISPSighting, self).from_dict(**kwargs) + + def __repr__(self): + if hasattr(self, 'value'): + return '<{self.__class__.__name__}(value={self.value})'.format(self=self) + return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) + + class MISPObjectAttribute(MISPAttribute): def __init__(self, definition): From 8013e90e40820dfbd254ff23bef9e481593a3d24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 27 Dec 2017 14:25:06 +0100 Subject: [PATCH 091/125] fix: Typo in set_sightings Fix #161 --- pymisp/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/api.py b/pymisp/api.py index 75e84b0..f79954f 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1301,7 +1301,7 @@ class PyMISP(object): sightings = [sightings] for sighting in sightings: if isinstance(sighting, MISPSighting): - to_post.appent(sighting.to_json()) + to_post.append(sighting.to_json()) elif isinstance(sighting, dict): to_post.append(json.dumps(sightings)) url = urljoin(self.root_url, 'sightings/add/') From 0ff21205118acd708de13bf241b46960db2ccf41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 27 Dec 2017 14:43:48 +0100 Subject: [PATCH 092/125] fix: Add method to add tags to objects Fix #160 --- pymisp/mispevent.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index abec80f..1a63bfe 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -785,6 +785,7 @@ class MISPObject(AbstractMISP): self.uuid = str(uuid.uuid4()) self.__fast_attribute_access = {} # Hashtable object_relation: [attributes] self.Attribute = [] + self.Tag = [] self._default_attributes_parameters = default_attributes_parameters if self._default_attributes_parameters: # Let's clean that up @@ -837,6 +838,9 @@ class MISPObject(AbstractMISP): if kwargs.get('ObjectReference'): for r in kwargs.pop('ObjectReference'): self.add_reference(**r) + if kwargs.get('Tag'): + for tag in kwargs.pop('Tag'): + self.add_tag(**tag) super(MISPObject, self).from_dict(**kwargs) @@ -854,6 +858,12 @@ class MISPObject(AbstractMISP): self.ObjectReference.append(reference) self.edited = True + def add_tag(self, name, **kwargs): + tag = MISPTag() + tag.from_dict(name=name, **kwargs) + self.Tag.append(tag) + self.edited = True + def get_attributes_by_relation(self, object_relation): '''Returns the list of attributes with the given object relation in the object''' return self.__fast_attribute_access.get(object_relation, []) From 72597c1b8aa84e8ddddaa271ba7b89a38a64b222 Mon Sep 17 00:00:00 2001 From: AninaAntonie Date: Thu, 28 Dec 2017 10:17:57 +0100 Subject: [PATCH 093/125] set_sightings Maybe I didn't use it correctly but the method set_sightings didn't work for me. It's working now but I'm not sure whether sending a request for every sighting in the list is the best solution. --- pymisp/api.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index f79954f..7c79542 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1295,17 +1295,16 @@ class PyMISP(object): return self._check_response(response) def set_sightings(self, sightings): - """Push a sighting (python dictionary)""" - to_post = [] + """Push a sighting (python dictionary or MISPSighting) or a list of sightings""" if not isinstance(sightings, list): sightings = [sightings] for sighting in sightings: if isinstance(sighting, MISPSighting): - to_post.append(sighting.to_json()) + to_post = sighting.to_json() elif isinstance(sighting, dict): - to_post.append(json.dumps(sightings)) - url = urljoin(self.root_url, 'sightings/add/') - response = self.__prepare_request('POST', url, json.dumps(to_post)) + to_post = json.dumps(sighting) + url = urljoin(self.root_url, 'sightings/add/') + response = self.__prepare_request('POST', url, to_post) return self._check_response(response) def sighting_per_json(self, json_file): From 6b81e5ddbaad634ec4d5937419009c8401a3e512 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 29 Dec 2017 14:42:49 +0100 Subject: [PATCH 094/125] fix: Properly set Tag to attributes within objects --- pymisp/mispevent.py | 26 +++++--------------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 1a63bfe..5d5d71f 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -169,11 +169,8 @@ class MISPAttribute(AbstractMISP): if kwargs.get('sharing_group_id'): self.sharing_group_id = int(kwargs.pop('sharing_group_id')) if kwargs.get('Tag'): - self.Tag = [] for tag in kwargs.pop('Tag'): - t = MISPTag() - t.from_dict(**tag) - self.Tag.append(t) + self.add_tag(**tag) # If the user wants to disable correlation, let them. Defaults to False. self.disable_correlation = kwargs.pop("disable_correlation", False) @@ -419,11 +416,7 @@ class MISPEvent(AbstractMISP): self.set_date(kwargs.pop('date')) if kwargs.get('Attribute'): for a in kwargs.pop('Attribute'): - attribute = MISPAttribute() - attribute.from_dict(**a) - if not hasattr(self, 'Attribute'): - self.Attribute = [] - self.Attribute.append(attribute) + self.add_attribute(**a) # All other keys if kwargs.get('id'): @@ -439,23 +432,16 @@ class MISPEvent(AbstractMISP): if kwargs.get('sharing_group_id'): self.sharing_group_id = int(kwargs.pop('sharing_group_id')) if kwargs.get('RelatedEvent'): - self.RelatedEvent = [] for rel_event in kwargs.pop('RelatedEvent'): sub_event = MISPEvent() sub_event.load(rel_event) self.RelatedEvent.append(sub_event) if kwargs.get('Tag'): - self.Tag = [] for tag in kwargs.pop('Tag'): - t = MISPTag() - t.from_dict(**tag) - self.Tag.append(t) + self.add_tag(**tag) if kwargs.get('Object'): - self.Object = [] for obj in kwargs.pop('Object'): - tmp_object = MISPObject(obj['name']) - tmp_object.from_dict(**obj) - self.Object.append(tmp_object) + self.add_object(**obj) super(MISPEvent, self).from_dict(**kwargs) @@ -543,8 +529,6 @@ class MISPEvent(AbstractMISP): else: attribute = MISPAttribute() attribute.from_dict(type=type, value=value, **kwargs) - if not hasattr(self, 'attributes'): - self.attributes = [] self.attributes.append(attribute) self.edited = True @@ -784,6 +768,7 @@ class MISPObject(AbstractMISP): pass self.uuid = str(uuid.uuid4()) self.__fast_attribute_access = {} # Hashtable object_relation: [attributes] + self.ObjectReference = [] self.Attribute = [] self.Tag = [] self._default_attributes_parameters = default_attributes_parameters @@ -805,7 +790,6 @@ class MISPObject(AbstractMISP): else: self.distribution = 5 # Default to inherit self.sharing_group_id = None - self.ObjectReference = [] self._standalone = standalone if self._standalone: # Mark as non_jsonable because we need to add the references manually after the object(s) have been created From c68b69b422480f626ddbc4486686050d7f7f4ef7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 29 Dec 2017 16:44:50 +0100 Subject: [PATCH 095/125] fix: Last commit was broken... --- pymisp/mispevent.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 5d5d71f..a7f2eeb 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -106,7 +106,7 @@ class MISPAttribute(AbstractMISP): """Mark the attribute as deleted (soft delete)""" self.deleted = True - def add_tag(self, tag): + def add_tag(self, **tag): """Add a tag to the attribute (by name or a MISPTag object)""" misp_tag = MISPTag() if isinstance(tag, str): @@ -457,7 +457,7 @@ class MISPEvent(AbstractMISP): to_return = {'Event': to_return} return to_return - def add_tag(self, tag): + def add_tag(self, **tag): """Add a tag to the attribute (by name or a MISPTag object)""" misp_tag = MISPTag() if isinstance(tag, str): @@ -539,7 +539,7 @@ class MISPEvent(AbstractMISP): return obj raise InvalidMISPObject('Object with {} does not exists in ths event'.format(object_id)) - def add_object(self, obj): + def add_object(self, **obj): """Add an object to the Event, either by passing a MISPObject, or a dictionary""" if isinstance(obj, MISPObject): self.Object.append(obj) @@ -721,8 +721,7 @@ class MISPObjectAttribute(MISPAttribute): if self.to_ids is None: # Same for the to_ids flag self.to_ids = self.__definition.get('to_ids') - kwargs.update(**self) - super(MISPObjectAttribute, self).from_dict(**kwargs) + super(MISPObjectAttribute, self).from_dict(**dict(self, **kwargs)) def __repr__(self): if hasattr(self, 'value'): From b183d3fd17842fdd7d4892e48f53c916c720885e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 3 Jan 2018 18:00:30 +0100 Subject: [PATCH 096/125] fix: the sharing_group_id isn't required. --- pymisp/mispevent.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index a7f2eeb..af3b85e 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -785,7 +785,8 @@ class MISPObject(AbstractMISP): self._default_attributes_parameters.pop('deleted', None) # doesn't make sense to pre-set it self._default_attributes_parameters.pop('data', None) # in case the original in a sample or an attachment self.distribution = self._default_attributes_parameters.distribution - self.sharing_group_id = self._default_attributes_parameters.sharing_group_id + if self._default_attributes_parameters.get('sharing_group_id'): + self.sharing_group_id = self._default_attributes_parameters.sharing_group_id else: self.distribution = 5 # Default to inherit self.sharing_group_id = None From 94cd2fd52eeebdc118045bdb834d2b047b432205 Mon Sep 17 00:00:00 2001 From: Arcuri Davide Date: Thu, 4 Jan 2018 08:53:43 +0100 Subject: [PATCH 097/125] _default_attributes_parameters - if set - is a dict Manage distribution and sharing_group_id as dict key like the other fields. -- Not sure about default --- pymisp/mispevent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index a7f2eeb..01444cc 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -784,8 +784,8 @@ class MISPObject(AbstractMISP): self._default_attributes_parameters.pop('category', None) # depends on the value self._default_attributes_parameters.pop('deleted', None) # doesn't make sense to pre-set it self._default_attributes_parameters.pop('data', None) # in case the original in a sample or an attachment - self.distribution = self._default_attributes_parameters.distribution - self.sharing_group_id = self._default_attributes_parameters.sharing_group_id + self.distribution = self._default_attributes_parameters.pop('distribution', 5) + self.sharing_group_id = self._default_attributes_parameters.pop('sharing_group_id', None) else: self.distribution = 5 # Default to inherit self.sharing_group_id = None From 4b6760536742f652ebfd40047aa09a9874037d49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 4 Jan 2018 10:58:24 +0100 Subject: [PATCH 098/125] chg: Allow do pass a category in default_attributes_parameters for object fix #166 --- pymisp/mispevent.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 4d5e6e5..3885b64 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -68,7 +68,11 @@ def _int_to_str(d): class MISPAttribute(AbstractMISP): - def __init__(self, describe_types=None): + def __init__(self, describe_types=None, strict=False): + """Represents an Attribute + :describe_type: Use it is you want to overwrite the defualt describeTypes.json file (you don't) + :strict: If false, fallback to sane defaults for the attribute type if the ones passed by the user are incorrect + """ super(MISPAttribute, self).__init__() if not describe_types: ressources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') @@ -79,6 +83,7 @@ class MISPAttribute(AbstractMISP): self._types = describe_types['types'] self.__category_type_mapping = describe_types['category_type_mappings'] self.__sane_default = describe_types['sane_defaults'] + self.__strict = strict self.Tag = [] @property @@ -121,8 +126,11 @@ class MISPAttribute(AbstractMISP): def from_dict(self, **kwargs): if kwargs.get('type') and kwargs.get('category'): if kwargs['type'] not in self.__category_type_mapping[kwargs['category']]: - raise NewAttributeError('{} and {} is an invalid combination, type for this category has to be in {}'.format( - kwargs.get('type'), kwargs.get('category'), (', '.join(self.__category_type_mapping[kwargs['category']])))) + if self.__strict: + raise NewAttributeError('{} and {} is an invalid combination, type for this category has to be in {}'.format( + kwargs.get('type'), kwargs.get('category'), (', '.join(self.__category_type_mapping[kwargs['category']])))) + else: + kwargs.pop('category', None) self.type = kwargs.pop('type', None) # Required if self.type is None: @@ -781,7 +789,6 @@ class MISPObject(AbstractMISP): self._default_attributes_parameters.pop('object_relation', None) # depends on the value self._default_attributes_parameters.pop('disable_correlation', None) # depends on the value self._default_attributes_parameters.pop('to_ids', None) # depends on the value - self._default_attributes_parameters.pop('category', None) # depends on the value self._default_attributes_parameters.pop('deleted', None) # doesn't make sense to pre-set it self._default_attributes_parameters.pop('data', None) # in case the original in a sample or an attachment self.distribution = self._default_attributes_parameters.pop('distribution', 5) From b9718c3fd389f13cc441835e10256cc760117719 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 4 Jan 2018 12:23:32 +0100 Subject: [PATCH 099/125] fix: Don't remove the distribution and sharing_group_id from default_attributes_parameters --- pymisp/mispevent.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 3885b64..5462a8f 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -791,11 +791,13 @@ class MISPObject(AbstractMISP): self._default_attributes_parameters.pop('to_ids', None) # depends on the value self._default_attributes_parameters.pop('deleted', None) # doesn't make sense to pre-set it self._default_attributes_parameters.pop('data', None) # in case the original in a sample or an attachment - self.distribution = self._default_attributes_parameters.pop('distribution', 5) - self.sharing_group_id = self._default_attributes_parameters.pop('sharing_group_id', 0) + + # Those values are set for the current object, if they exist, but not pop'd because they are still useful for the attributes + self.distribution = self._default_attributes_parameters.get('distribution', 5) + self.sharing_group_id = self._default_attributes_parameters.get('sharing_group_id', 0) else: self.distribution = 5 # Default to inherit - self.sharing_group_id = None + self.sharing_group_id = 0 self._standalone = standalone if self._standalone: # Mark as non_jsonable because we need to add the references manually after the object(s) have been created From bb1aac5720bdc6be90ded6bcd5e03e772d56f958 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 3 Jan 2018 14:36:10 +0100 Subject: [PATCH 100/125] chg: Multiple changes * Fix timestamp dump (properly enforce UTC) * Properly handle proposals * Add many getter/setter * Add dedicated test cases for MISPEvent and other objects --- .travis.yml | 2 +- pymisp/abstract.py | 25 +- pymisp/mispevent.py | 265 +- tests/mispevent_testfiles/attribute.json | 23 + tests/mispevent_testfiles/attribute_del.json | 25 + tests/mispevent_testfiles/event.json | 10 + .../event_obj_attr_tag.json | 59 + .../event_obj_def_param.json | 56 + tests/mispevent_testfiles/event_obj_tag.json | 31 + tests/mispevent_testfiles/existing_event.json | 4573 +++++++++++++++++ tests/mispevent_testfiles/malware.json | 21 + tests/mispevent_testfiles/proposals.json | 36 + tests/mispevent_testfiles/shadow.json | 149 + tests/mispevent_testfiles/sighting.json | 5 + tests/mispevent_testfiles/simple.json | 4 + tests/test_mispevent.py | 137 + 16 files changed, 5367 insertions(+), 54 deletions(-) create mode 100644 tests/mispevent_testfiles/attribute.json create mode 100644 tests/mispevent_testfiles/attribute_del.json create mode 100644 tests/mispevent_testfiles/event.json create mode 100644 tests/mispevent_testfiles/event_obj_attr_tag.json create mode 100644 tests/mispevent_testfiles/event_obj_def_param.json create mode 100644 tests/mispevent_testfiles/event_obj_tag.json create mode 100644 tests/mispevent_testfiles/existing_event.json create mode 100644 tests/mispevent_testfiles/malware.json create mode 100644 tests/mispevent_testfiles/proposals.json create mode 100644 tests/mispevent_testfiles/shadow.json create mode 100644 tests/mispevent_testfiles/sighting.json create mode 100644 tests/mispevent_testfiles/simple.json create mode 100644 tests/test_mispevent.py diff --git a/.travis.yml b/.travis.yml index 93ef5eb..0bd5639 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ install: - popd script: - - nosetests --with-coverage --cover-package=pymisp tests/test_offline.py + - nosetests --with-coverage --cover-package=pymisp tests/test_*.py after_success: - codecov diff --git a/pymisp/abstract.py b/pymisp/abstract.py index de0c40d..668a17f 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -15,6 +15,21 @@ logger = logging.getLogger('pymisp') if six.PY2: logger.warning("You're using python 2, it is strongly recommended to use python >=3.5") + # This is required because Python 2 is a pain. + from datetime import tzinfo, timedelta + + class UTC(tzinfo): + """UTC""" + + def utcoffset(self, dt): + return timedelta(0) + + def tzname(self, dt): + return "UTC" + + def dst(self, dt): + return timedelta(0) + class MISPEncode(JSONEncoder): @@ -80,6 +95,8 @@ class AbstractMISP(collections.MutableMapping): val = getattr(self, attribute, None) if val is None: continue + elif isinstance(val, list) and len(val) == 0: + continue if attribute == 'timestamp': if self.edited: # In order to be accepted by MISP, the timestamp of an object @@ -98,7 +115,7 @@ class AbstractMISP(collections.MutableMapping): def to_json(self): """Dump recursively any class of type MISPAbstract to a json string""" - return json.dumps(self, cls=MISPEncode) + return json.dumps(self, cls=MISPEncode, sort_keys=True, indent=2) def __getitem__(self, key): try: @@ -150,10 +167,10 @@ class AbstractMISP(collections.MutableMapping): def _datetime_to_timestamp(self, d): """Convert a datetime.datetime object to a timestamp (int)""" - if isinstance(d, (int, str)): + if isinstance(d, (int, str)) or (sys.version_info < (3, 0) and isinstance(d, unicode)): # Assume we already have a timestamp return d if sys.version_info >= (3, 3): - return d.timestamp() + return int(d.timestamp()) else: - return (d - datetime.datetime.utcfromtimestamp(0)).total_seconds() + return int((d - datetime.datetime.fromtimestamp(0, UTC())).total_seconds()) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index a7f2eeb..6223b4f 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -25,6 +25,21 @@ logger = logging.getLogger('pymisp') if six.PY2: logger.warning("You're using python 2, it is strongly recommended to use python >=3.5") + # This is required because Python 2 is a pain. + from datetime import tzinfo, timedelta + + class UTC(tzinfo): + """UTC""" + + def utcoffset(self, dt): + return timedelta(0) + + def tzname(self, dt): + return "UTC" + + def dst(self, dt): + return timedelta(0) + try: from dateutil.parser import parse except ImportError: @@ -80,44 +95,86 @@ class MISPAttribute(AbstractMISP): self.__category_type_mapping = describe_types['category_type_mappings'] self.__sane_default = describe_types['sane_defaults'] self.Tag = [] + self.ShadowAttribute = [] @property def known_types(self): + """Returns a list of all the known MISP attributes types""" return self._types @property def malware_binary(self): + """Returns a BytesIO of the malware (if the attribute has one, obvs).""" if hasattr(self, '_malware_binary'): return self._malware_binary return None @property def tags(self): + """Returns a lost of tags associated to this Attribute""" return self.Tag @tags.setter def tags(self, tags): + """Set a list of prepared MISPTag.""" if all(isinstance(x, MISPTag) for x in tags): self.Tag = tags else: - raise PyMISPError('All the attributes have to be of type MISPAttribute.') + raise PyMISPError('All the attributes have to be of type MISPTag.') + + @property + def shadow_attributes(self): + return self.ShadowAttribute + + @shadow_attributes.setter + def shadow_attributes(self, shadow_attributes): + """Set a list of prepared MISPShadowAttribute.""" + if all(isinstance(x, MISPShadowAttribute) for x in shadow_attributes): + self.ShadowAttribute = shadow_attributes + else: + raise PyMISPError('All the attributes have to be of type MISPShadowAttribute.') def delete(self): """Mark the attribute as deleted (soft delete)""" self.deleted = True - def add_tag(self, **tag): + def add_tag(self, tag=None, **kwargs): """Add a tag to the attribute (by name or a MISPTag object)""" - misp_tag = MISPTag() if isinstance(tag, str): + misp_tag = MISPTag() misp_tag.from_dict(name=tag) + elif isinstance(tag, MISPTag): + misp_tag = tag elif isinstance(tag, dict): + misp_tag = MISPTag() misp_tag.from_dict(**tag) + elif kwargs: + misp_tag = MISPTag() + misp_tag.from_dict(**kwargs) else: - raise PyMISPError("The tag is in an invalid format (can be either string, or list): {}".format(tag)) + raise PyMISPError("The tag is in an invalid format (can be either string, MISPTag, or an expanded dict): {}".format(tag)) self.tags.append(misp_tag) self.edited = True + def add_proposal(self, shadow_attribute=None, **kwargs): + """Alias for add_shadow_attribute""" + self.add_shadow_attribute(shadow_attribute, **kwargs) + + def add_shadow_attribute(self, shadow_attribute=None, **kwargs): + """Add a tag to the attribute (by name or a MISPTag object)""" + if isinstance(shadow_attribute, MISPShadowAttribute): + misp_shadow_attribute = shadow_attribute + elif isinstance(shadow_attribute, dict): + misp_shadow_attribute = MISPShadowAttribute() + misp_shadow_attribute.from_dict(**shadow_attribute) + elif kwargs: + misp_shadow_attribute = MISPShadowAttribute() + misp_shadow_attribute.from_dict(**kwargs) + else: + raise PyMISPError("The shadow_attribute is in an invalid format (can be either string, MISPShadowAttribute, or an expanded dict): {}".format(shadow_attribute)) + self.shadow_attributes.append(misp_shadow_attribute) + self.edited = True + def from_dict(self, **kwargs): if kwargs.get('type') and kwargs.get('category'): if kwargs['type'] not in self.__category_type_mapping[kwargs['category']]: @@ -165,12 +222,18 @@ class MISPAttribute(AbstractMISP): if kwargs.get('event_id'): self.event_id = int(kwargs.pop('event_id')) if kwargs.get('timestamp'): - self.timestamp = datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=int(kwargs.pop('timestamp'))) + if sys.version_info >= (3, 3): + self.timestamp = datetime.datetime.fromtimestamp(int(kwargs.pop('timestamp')), datetime.timezone.utc) + else: + self.timestamp = datetime.datetime.fromtimestamp(int(kwargs.pop('timestamp')), UTC()) if kwargs.get('sharing_group_id'): self.sharing_group_id = int(kwargs.pop('sharing_group_id')) if kwargs.get('Tag'): for tag in kwargs.pop('Tag'): - self.add_tag(**tag) + self.add_tag(tag) + if kwargs.get('ShadowAttribute'): + for s_attr in kwargs.pop('ShadowAttribute'): + self.add_shadow_attribute(s_attr) # If the user wants to disable correlation, let them. Defaults to False. self.disable_correlation = kwargs.pop("disable_correlation", False) @@ -240,7 +303,7 @@ class MISPAttribute(AbstractMISP): return '<{self.__class__.__name__}(type={self.type}, value={self.value})'.format(self=self) return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) - def verify(self, gpg_uid): + def verify(self, gpg_uid): # pragma: no cover # Not used if not has_pyme: raise PyMISPError('pyme is required, please install: pip install --pre pyme3. You will also need libgpg-error-dev and libgpgme11-dev.') @@ -253,13 +316,13 @@ class MISPAttribute(AbstractMISP): except Exception: return {self.uuid: False} - def _serialize(self): + def _serialize(self): # pragma: no cover # Not used return '{type}{category}{to_ids}{uuid}{timestamp}{comment}{deleted}{value}'.format( type=self.type, category=self.category, to_ids=self.to_ids, uuid=self.uuid, timestamp=self.timestamp, comment=self.comment, deleted=self.deleted, value=self.value).encode() - def sign(self, gpg_uid, passphrase=None): + def sign(self, gpg_uid, passphrase=None): # pragma: no cover # Not used if not has_pyme: raise PyMISPError('pyme is required, please install: pip install --pre pyme3. You will also need libgpg-error-dev and libgpgme11-dev.') @@ -273,23 +336,23 @@ class MISPAttribute(AbstractMISP): self.sig = base64.b64encode(signed).decode() @deprecated - def get_known_types(self): + def get_known_types(self): # pragma: no cover return self.known_types @deprecated - def get_malware_binary(self): + def get_malware_binary(self): # pragma: no cover return self.malware_binary @deprecated - def _json(self): + def _json(self): # pragma: no cover return self.to_dict() @deprecated - def _json_full(self): + def _json_full(self): # pragma: no cover return self.to_dict() @deprecated - def set_all_values(self, **kwargs): + def set_all_values(self, **kwargs): # pragma: no cover self.from_dict(**kwargs) @@ -314,6 +377,7 @@ class MISPEvent(AbstractMISP): self.Attribute = [] self.Object = [] self.RelatedEvent = [] + self.ShadowAttribute = [] @property def known_types(self): @@ -330,6 +394,17 @@ class MISPEvent(AbstractMISP): else: raise PyMISPError('All the attributes have to be of type MISPAttribute.') + @property + def shadow_attributes(self): + return self.ShadowAttribute + + @shadow_attributes.setter + def shadow_attributes(self, shadow_attributes): + if all(isinstance(x, MISPShadowAttribute) for x in shadow_attributes): + self.ShadowAttribute = shadow_attributes + else: + raise PyMISPError('All the attributes have to be of type MISPShadowAttribute.') + @property def related_events(self): return self.RelatedEvent @@ -338,10 +413,24 @@ class MISPEvent(AbstractMISP): def objects(self): return self.Object + @objects.setter + def objects(self, objects): + if all(isinstance(x, MISPObject) for x in objects): + self.Object = objects + else: + raise PyMISPError('All the attributes have to be of type MISPObject.') + @property def tags(self): return self.Tag + @tags.setter + def tags(self, tags): + if all(isinstance(x, MISPTag) for x in tags): + self.Tag = tags + else: + raise PyMISPError('All the attributes have to be of type MISPTag.') + def load_file(self, event_path): """Load a JSON dump from a file on the disk""" if not os.path.exists(event_path): @@ -363,7 +452,9 @@ class MISPEvent(AbstractMISP): if not event: raise PyMISPError('Invalid event') # Invalid event created by MISP up to 2.4.52 (attribute_count is none instead of '0') - if event.get('Event') and event.get('Event').get('attribute_count') is None: + if (event.get('Event') and + 'attribute_count' in event.get('Event') and + event.get('Event').get('attribute_count') is None): event['Event']['attribute_count'] = '0' jsonschema.validate(event, self.__json_schema) e = event.get('Event') @@ -426,9 +517,15 @@ class MISPEvent(AbstractMISP): if kwargs.get('org_id'): self.org_id = int(kwargs.pop('org_id')) if kwargs.get('timestamp'): - self.timestamp = datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=int(kwargs.pop('timestamp'))) + if sys.version_info >= (3, 3): + self.timestamp = datetime.datetime.fromtimestamp(int(kwargs.pop('timestamp')), datetime.timezone.utc) + else: + self.timestamp = datetime.datetime.fromtimestamp(int(kwargs.pop('timestamp')), UTC()) if kwargs.get('publish_timestamp'): - self.publish_timestamp = datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=int(kwargs.pop('publish_timestamp'))) + if sys.version_info >= (3, 3): + self.publish_timestamp = datetime.datetime.fromtimestamp(int(kwargs.pop('publish_timestamp')), datetime.timezone.utc) + else: + self.publish_timestamp = datetime.datetime.fromtimestamp(int(kwargs.pop('publish_timestamp')), UTC()) if kwargs.get('sharing_group_id'): self.sharing_group_id = int(kwargs.pop('sharing_group_id')) if kwargs.get('RelatedEvent'): @@ -438,10 +535,10 @@ class MISPEvent(AbstractMISP): self.RelatedEvent.append(sub_event) if kwargs.get('Tag'): for tag in kwargs.pop('Tag'): - self.add_tag(**tag) + self.add_tag(tag) if kwargs.get('Object'): for obj in kwargs.pop('Object'): - self.add_object(**obj) + self.add_object(obj) super(MISPEvent, self).from_dict(**kwargs) @@ -449,6 +546,8 @@ class MISPEvent(AbstractMISP): to_return = super(MISPEvent, self).to_dict() if to_return.get('date'): + if isinstance(self.date, datetime.datetime): + self.date = self.date.date() to_return['date'] = self.date.isoformat() if to_return.get('publish_timestamp'): to_return['publish_timestamp'] = self._datetime_to_timestamp(self.publish_timestamp) @@ -457,15 +556,40 @@ class MISPEvent(AbstractMISP): to_return = {'Event': to_return} return to_return - def add_tag(self, **tag): + def add_proposal(self, shadow_attribute=None, **kwargs): + """Alias for add_shadow_attribute""" + self.add_shadow_attribute(shadow_attribute, **kwargs) + + def add_shadow_attribute(self, shadow_attribute=None, **kwargs): """Add a tag to the attribute (by name or a MISPTag object)""" - misp_tag = MISPTag() - if isinstance(tag, str): - misp_tag.from_dict(name=tag) - elif isinstance(tag, dict): - misp_tag.from_dict(**tag) + if isinstance(shadow_attribute, MISPShadowAttribute): + misp_shadow_attribute = shadow_attribute + elif isinstance(shadow_attribute, dict): + misp_shadow_attribute = MISPShadowAttribute() + misp_shadow_attribute.from_dict(**shadow_attribute) + elif kwargs: + misp_shadow_attribute = MISPShadowAttribute() + misp_shadow_attribute.from_dict(**kwargs) else: - raise PyMISPError("The tag is in an invalid format (can be either string, or list): {}".format(tag)) + raise PyMISPError("The shadow_attribute is in an invalid format (can be either string, MISPShadowAttribute, or an expanded dict): {}".format(shadow_attribute)) + self.shadow_attributes.append(misp_shadow_attribute) + self.edited = True + + def add_tag(self, tag=None, **kwargs): + """Add a tag to the attribute (by name or a MISPTag object)""" + if isinstance(tag, str): + misp_tag = MISPTag() + misp_tag.from_dict(name=tag) + elif isinstance(tag, MISPTag): + misp_tag = tag + elif isinstance(tag, dict): + misp_tag = MISPTag() + misp_tag.from_dict(**tag) + elif kwargs: + misp_tag = MISPTag() + misp_tag.from_dict(**kwargs) + else: + raise PyMISPError("The tag is in an invalid format (can be either string, MISPTag, or an expanded dict): {}".format(tag)) self.tags.append(misp_tag) self.edited = True @@ -484,7 +608,7 @@ class MISPEvent(AbstractMISP): def add_attribute_tag(self, tag, attribute_identifier): '''Add a tag to an existing attribute, raise an Exception if the attribute doesn't exists. - :tag: Tag name + :tag: Tag name as a string, MISPTag instance, or dictionary :attribute_identifier: can be an ID, UUID, or the value. ''' attributes = [] @@ -539,16 +663,23 @@ class MISPEvent(AbstractMISP): return obj raise InvalidMISPObject('Object with {} does not exists in ths event'.format(object_id)) - def add_object(self, **obj): + def add_object(self, obj=None, **kwargs): """Add an object to the Event, either by passing a MISPObject, or a dictionary""" if isinstance(obj, MISPObject): - self.Object.append(obj) + misp_obj = obj elif isinstance(obj, dict): - tmp_object = MISPObject(obj['name']) - tmp_object.from_dict(**obj) - self.Object.append(tmp_object) + misp_obj = MISPObject(name=obj.pop('name'), strict=obj.pop('strict', False), + default_attributes_parameters=obj.pop('default_attributes_parameters', {}), + **obj) + misp_obj.from_dict(**obj) + elif kwargs: + misp_obj = MISPObject(name=kwargs.pop('name'), strict=kwargs.pop('strict', False), + default_attributes_parameters=kwargs.pop('default_attributes_parameters', {}), + **kwargs) + misp_obj.from_dict(**kwargs) else: raise InvalidMISPObject("An object to add to an existing Event needs to be either a MISPObject, or a plain python dictionary") + self.Object.append(misp_obj) self.edited = True def __repr__(self): @@ -561,14 +692,14 @@ class MISPEvent(AbstractMISP): date=self.date, threat_level_id=self.threat_level_id, info=self.info, uuid=self.uuid, analysis=self.analysis, timestamp=self.timestamp).encode() - def _serialize_sigs(self): + def _serialize_sigs(self): # pragma: no cover # Not used all_sigs = self.sig for a in self.attributes: all_sigs += a.sig return all_sigs.encode() - def sign(self, gpg_uid, passphrase=None): + def sign(self, gpg_uid, passphrase=None): # pragma: no cover # Not used if not has_pyme: raise PyMISPError('pyme is required, please install: pip install --pre pyme3. You will also need libgpg-error-dev and libgpgme11-dev.') @@ -591,7 +722,7 @@ class MISPEvent(AbstractMISP): signed, _ = c.sign(to_sign_global, mode=mode.DETACH) self.global_sig = base64.b64encode(signed).decode() - def verify(self, gpg_uid): + def verify(self, gpg_uid): # pragma: no cover # Not used if not has_pyme: raise PyMISPError('pyme is required, please install: pip install --pre pyme3. You will also need libgpg-error-dev and libgpgme11-dev.') @@ -617,15 +748,15 @@ class MISPEvent(AbstractMISP): return to_return @deprecated - def get_known_types(self): + def get_known_types(self): # pragma: no cover return self.known_types @deprecated - def set_all_values(self, **kwargs): + def set_all_values(self, **kwargs): # pragma: no cover self.from_dict(**kwargs) @deprecated - def _json(self): + def _json(self): # pragma: no cover return self.to_dict() @@ -712,7 +843,7 @@ class MISPObjectAttribute(MISPAttribute): self.type = kwargs.pop('type', None) if self.type is None: self.type = self.__definition.get('misp-attribute') - self.disable_correlation = kwargs.pop('disable_correlation', None) + self.disable_correlation = kwargs.pop('disable_correlation', False) if self.disable_correlation is None: # The correlation can be disabled by default in the object definition. # Use this value if it isn't overloaded by the object @@ -729,6 +860,12 @@ class MISPObjectAttribute(MISPAttribute): return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) +class MISPShadowAttribute(MISPAttribute): + + def __init__(self): + super(MISPShadowAttribute, self).__init__() + + class MISPObject(AbstractMISP): def __init__(self, name, strict=False, standalone=False, default_attributes_parameters={}, **kwargs): @@ -769,8 +906,12 @@ class MISPObject(AbstractMISP): self.__fast_attribute_access = {} # Hashtable object_relation: [attributes] self.ObjectReference = [] self.Attribute = [] - self.Tag = [] - self._default_attributes_parameters = default_attributes_parameters + # self.Tag = [] See https://github.com/MISP/PyMISP/issues/168 + if isinstance(default_attributes_parameters, MISPAttribute): + # Just make sure we're not modifying an existing MISPAttribute + self._default_attributes_parameters = default_attributes_parameters.to_dict() + else: + self._default_attributes_parameters = default_attributes_parameters if self._default_attributes_parameters: # Let's clean that up self._default_attributes_parameters.pop('value', None) # duh @@ -821,9 +962,11 @@ class MISPObject(AbstractMISP): if kwargs.get('ObjectReference'): for r in kwargs.pop('ObjectReference'): self.add_reference(**r) - if kwargs.get('Tag'): - for tag in kwargs.pop('Tag'): - self.add_tag(**tag) + + # Not supported yet - https://github.com/MISP/PyMISP/issues/168 + # if kwargs.get('Tag'): + # for tag in kwargs.pop('Tag'): + # self.add_tag(tag) super(MISPObject, self).from_dict(**kwargs) @@ -841,11 +984,35 @@ class MISPObject(AbstractMISP): self.ObjectReference.append(reference) self.edited = True - def add_tag(self, name, **kwargs): - tag = MISPTag() - tag.from_dict(name=name, **kwargs) - self.Tag.append(tag) - self.edited = True + # Not supported yet - https://github.com/MISP/PyMISP/issues/168 + # @property + # def tags(self): + # return self.Tag + + # @tags.setter + # def tags(self, tags): + # if all(isinstance(x, MISPTag) for x in tags): + # self.Tag = tags + # else: + # raise PyMISPError('All the attributes have to be of type MISPTag.') + + # def add_tag(self, tag=None, **kwargs): + # """Add a tag to the attribute (by name or a MISPTag object)""" + # if isinstance(tag, str): + # misp_tag = MISPTag() + # misp_tag.from_dict(name=tag) + # elif isinstance(tag, MISPTag): + # misp_tag = tag + # elif isinstance(tag, dict): + # misp_tag = MISPTag() + # misp_tag.from_dict(**tag) + # elif kwargs: + # misp_tag = MISPTag() + # misp_tag.from_dict(**kwargs) + # else: + # raise PyMISPError("The tag is in an invalid format (can be either string, MISPTag, or an expanded dict): {}".format(tag)) + # self.tags.append(misp_tag) + # self.edited = True def get_attributes_by_relation(self, object_relation): '''Returns the list of attributes with the given object relation in the object''' diff --git a/tests/mispevent_testfiles/attribute.json b/tests/mispevent_testfiles/attribute.json new file mode 100644 index 0000000..8ad4843 --- /dev/null +++ b/tests/mispevent_testfiles/attribute.json @@ -0,0 +1,23 @@ +{ + "Event": { + "Attribute": [ + { + "Tag": [ + { + "name": "osint" + } + ], + "category": "Payload delivery", + "disable_correlation": false, + "to_ids": true, + "type": "filename", + "value": "bar.exe" + } + ], + "analysis": "1", + "date": "2017-12-31", + "distribution": "1", + "info": "This is a test", + "threat_level_id": "1" + } +} diff --git a/tests/mispevent_testfiles/attribute_del.json b/tests/mispevent_testfiles/attribute_del.json new file mode 100644 index 0000000..d381cfe --- /dev/null +++ b/tests/mispevent_testfiles/attribute_del.json @@ -0,0 +1,25 @@ +{ + "Event": { + "Attribute": [ + { + "Tag": [ + { + "name": "osint" + } + ], + "category": "Payload delivery", + "deleted": true, + "disable_correlation": false, + "id": "42", + "to_ids": true, + "type": "filename", + "value": "bar.exe" + } + ], + "analysis": "1", + "date": "2017-12-31", + "distribution": "1", + "info": "This is a test", + "threat_level_id": "1" + } +} diff --git a/tests/mispevent_testfiles/event.json b/tests/mispevent_testfiles/event.json new file mode 100644 index 0000000..0dcc796 --- /dev/null +++ b/tests/mispevent_testfiles/event.json @@ -0,0 +1,10 @@ +{ + "Event": { + "analysis": "1", + "date": "2017-12-31", + "distribution": "1", + "info": "This is a test", + "published": true, + "threat_level_id": "1" + } +} diff --git a/tests/mispevent_testfiles/event_obj_attr_tag.json b/tests/mispevent_testfiles/event_obj_attr_tag.json new file mode 100644 index 0000000..7d336a7 --- /dev/null +++ b/tests/mispevent_testfiles/event_obj_attr_tag.json @@ -0,0 +1,59 @@ +{ + "Event": { + "Object": [ + { + "Attribute": [ + { + "Tag": [ + { + "name": "blah" + } + ], + "category": "Payload delivery", + "disable_correlation": false, + "object_relation": "filename", + "to_ids": true, + "type": "filename", + "value": "bar" + } + ], + "ObjectReference": [ + { + "comment": "foo", + "object_uuid": "a", + "referenced_uuid": "b", + "relationship_type": "baz" + } + ], + "description": "File object describing a file with meta-information", + "distribution": 5, + "meta-category": "file", + "name": "file", + "sharing_group_id": 0, + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": 8, + "uuid": "a" + }, + { + "Attribute": [ + { + "category": "External analysis", + "disable_correlation": false, + "object_relation": "url", + "to_ids": true, + "type": "url", + "value": "https://www.circl.lu" + } + ], + "description": "url object describes an url along with its normalized field (like extracted using faup parsing library) and its metadata.", + "distribution": 5, + "meta-category": "network", + "name": "url", + "sharing_group_id": 0, + "template_uuid": "60efb77b-40b5-4c46-871b-ed1ed999fce5", + "template_version": 5, + "uuid": "b" + } + ] + } +} diff --git a/tests/mispevent_testfiles/event_obj_def_param.json b/tests/mispevent_testfiles/event_obj_def_param.json new file mode 100644 index 0000000..559ba89 --- /dev/null +++ b/tests/mispevent_testfiles/event_obj_def_param.json @@ -0,0 +1,56 @@ +{ + "Event": { + "Object": [ + { + "Attribute": [ + { + "Tag": [ + { + "name": "blah" + } + ], + "category": "Payload delivery", + "disable_correlation": false, + "object_relation": "filename", + "to_ids": true, + "type": "filename", + "value": "bar" + } + ], + "description": "File object describing a file with meta-information", + "distribution": 5, + "meta-category": "file", + "name": "file", + "sharing_group_id": 0, + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": 8, + "uuid": "a" + }, + { + "Attribute": [ + { + "Tag": [ + { + "name": "blah" + } + ], + "category": "Payload delivery", + "disable_correlation": false, + "object_relation": "filename", + "to_ids": true, + "type": "filename", + "value": "baz" + } + ], + "description": "File object describing a file with meta-information", + "distribution": 5, + "meta-category": "file", + "name": "file", + "sharing_group_id": 0, + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": 8, + "uuid": "b" + } + ] + } +} diff --git a/tests/mispevent_testfiles/event_obj_tag.json b/tests/mispevent_testfiles/event_obj_tag.json new file mode 100644 index 0000000..736bff2 --- /dev/null +++ b/tests/mispevent_testfiles/event_obj_tag.json @@ -0,0 +1,31 @@ +{ + "Event": { + "Object": [ + { + "Attribute": [ + { + "category": "Payload delivery", + "disable_correlation": false, + "object_relation": "filename", + "to_ids": true, + "type": "filename", + "value": "bar" + } + ], + "Tag": [ + { + "name": "osint" + } + ], + "description": "File object describing a file with meta-information", + "distribution": 5, + "meta-category": "file", + "name": "file", + "sharing_group_id": 0, + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": 8, + "uuid": "a" + } + ] + } +} diff --git a/tests/mispevent_testfiles/existing_event.json b/tests/mispevent_testfiles/existing_event.json new file mode 100644 index 0000000..d140132 --- /dev/null +++ b/tests/mispevent_testfiles/existing_event.json @@ -0,0 +1,4573 @@ +{ + "Event": { + "Attribute": [ + { + "Tag": [ + { + "colour": "#00223b", + "exportable": true, + "hide_tag": false, + "id": "101", + "name": "osint:source-type=\"blog-post\"", + "user_id": "0" + }, + { + "colour": "#007cd6", + "exportable": true, + "hide_tag": false, + "id": "618", + "name": "osint:certainty=\"93\"", + "user_id": "0" + } + ], + "category": "External analysis", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188757", + "object_id": "0", + "sharing_group_id": "0", + "timestamp": "1513893921", + "to_ids": false, + "type": "link", + "uuid": "5a3c2fda-78f4-44b7-8366-46da02de0b81", + "value": "https://www.welivesecurity.com/2017/12/21/sednit-update-fancy-bear-spent-year/" + }, + { + "Tag": [ + { + "colour": "#00223b", + "exportable": true, + "hide_tag": false, + "id": "101", + "name": "osint:source-type=\"blog-post\"", + "user_id": "0" + }, + { + "colour": "#007cd6", + "exportable": true, + "hide_tag": false, + "id": "618", + "name": "osint:certainty=\"93\"", + "user_id": "0" + } + ], + "category": "External analysis", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188758", + "object_id": "0", + "sharing_group_id": "0", + "timestamp": "1513893921", + "to_ids": false, + "type": "text", + "uuid": "5a3c2fee-7c8c-438a-8f7f-465402de0b81", + "value": "The Sednit group \u2014 also known as Strontium, APT28, Fancy Bear or Sofacy\u2009\u2014\u2009is a group of attackers operating since 2004, if not earlier, and whose main objective is to steal confidential information from specific targets.\r\n\r\nThis article is a follow-up to ESET\u2019s presentation at BlueHat in November 2017. Late in 2016 we published a white paper covering Sednit activity between 2014 and 2016. Since then, we have continued to actively track Sednit\u2019s operations, and today we are publishing a brief overview of what our tracking uncovered in terms of the group\u2019s activities and updates to their toolset. The first section covers the update of their attack methodology: namely, the ways in which this group tries to compromise their targets systems. The second section covers the evolution of their tools, with a particular emphasis on a detailed analysis of a new version of their flagship malware: Xagent." + }, + { + "category": "Network activity", + "comment": "Xagent Samples", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188759", + "object_id": "0", + "sharing_group_id": "0", + "timestamp": "1513893957", + "to_ids": true, + "type": "domain", + "uuid": "5a3c3045-ab0c-4d38-8efe-459002de0b81", + "value": "movieultimate.com" + }, + { + "category": "Network activity", + "comment": "Xagent Samples", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188760", + "object_id": "0", + "sharing_group_id": "0", + "timestamp": "1513893957", + "to_ids": true, + "type": "domain", + "uuid": "5a3c3045-61dc-495c-ae8a-471e02de0b81", + "value": "meteost.com" + }, + { + "category": "Network activity", + "comment": "Xagent Samples", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188761", + "object_id": "0", + "sharing_group_id": "0", + "timestamp": "1513893957", + "to_ids": true, + "type": "domain", + "uuid": "5a3c3045-e354-4978-a6b4-49ad02de0b81", + "value": "faststoragefiles.org" + }, + { + "category": "Network activity", + "comment": "Xagent Samples", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188762", + "object_id": "0", + "sharing_group_id": "0", + "timestamp": "1513893957", + "to_ids": true, + "type": "domain", + "uuid": "5a3c3045-968c-4572-9f64-491502de0b81", + "value": "nethostnet.com" + }, + { + "category": "Network activity", + "comment": "Xagent Samples", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188763", + "object_id": "0", + "sharing_group_id": "0", + "timestamp": "1513893957", + "to_ids": true, + "type": "domain", + "uuid": "5a3c3045-eb44-433f-a13a-44b902de0b81", + "value": "fsportal.net" + }, + { + "category": "Network activity", + "comment": "Xagent Samples", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188764", + "object_id": "0", + "sharing_group_id": "0", + "timestamp": "1513893957", + "to_ids": true, + "type": "domain", + "uuid": "5a3c3045-6a88-479d-b799-4d3d02de0b81", + "value": "fastdataexchange.org" + }, + { + "category": "Network activity", + "comment": "Xagent Samples", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188765", + "object_id": "0", + "sharing_group_id": "0", + "timestamp": "1513893957", + "to_ids": true, + "type": "domain", + "uuid": "5a3c3045-7480-4831-a5c4-48c802de0b81", + "value": "newfilmts.com" + } + ], + "Galaxy": [ + { + "GalaxyCluster": [ + { + "authors": [ + "Alexandre Dulaunoy", + "Florian Roth", + "Thomas Schreck", + "Timo Steffens", + "Various" + ], + "description": "The Sofacy Group (also known as APT28, Pawn Storm, Fancy Bear and Sednit) is a cyber espionage group believed to have ties to the Russian government. Likely operating since 2007, the group is known to target government, military, and security organizations. It has been characterized as an advanced persistent threat.", + "galaxy_id": "366", + "id": "45563", + "meta": { + "country": [ + "RU" + ], + "refs": [ + "https://en.wikipedia.org/wiki/Sofacy_Group", + "https://aptnotes.malwareconfig.com/web/viewer.html?file=../APTnotes/2014/apt28.pdf", + "http://www.trendmicro.com/cloud-content/us/pdfs/security-intelligence/white-papers/wp-operation-pawn-storm.pdf", + "https://www2.fireeye.com/rs/848-DID-242/images/wp-mandiant-matryoshka-mining.pdf", + "https://www.crowdstrike.com/blog/bears-midst-intrusion-democratic-national-committee/", + "http://researchcenter.paloaltonetworks.com/2016/06/unit42-new-sofacy-attacks-against-us-government-agency/" + ], + "synonyms": [ + "APT 28", + "APT28", + "Pawn Storm", + "Fancy Bear", + "Sednit", + "TsarTeam", + "TG-4127", + "Group-4127", + "STRONTIUM", + "TAG_0700", + "Swallowtail", + "IRON TWILIGHT", + "Group 74" + ] + }, + "source": "MISP Project", + "tag_id": "1100", + "tag_name": "misp-galaxy:threat-actor=\"Sofacy\"", + "type": "threat-actor", + "uuid": "7cdff317-a673-4474-84ec-4f1754947823", + "value": "Sofacy", + "version": "30" + } + ], + "description": "Threat actors are characteristics of malicious actors (or adversaries) representing a cyber attack threat including presumed intent and historically observed behaviour.", + "icon": "user-secret", + "id": "366", + "name": "Threat Actor", + "type": "threat-actor", + "uuid": "698774c7-8022-42c4-917f-8d6e4f06ada3", + "version": "2" + }, + { + "GalaxyCluster": [ + { + "authors": [ + "Kafeine", + "Will Metcalf", + "KahuSecurity" + ], + "description": "Sednit EK is the exploit kit used by APT28", + "galaxy_id": "370", + "id": "38813", + "meta": { + "refs": [ + "http://www.welivesecurity.com/2014/10/08/sednit-espionage-group-now-using-custom-exploit-kit/", + "http://blog.trendmicro.com/trendlabs-security-intelligence/new-adobe-flash-zero-day-used-in-pawn-storm-campaign/" + ], + "status": [ + "Active" + ] + }, + "source": "MISP Project", + "tag_id": "3007", + "tag_name": "misp-galaxy:exploit-kit=\"Sednit EK\"", + "type": "exploit-kit", + "uuid": "454f4e78-bd7c-11e6-a4a6-cec0c932ce01", + "value": "Sednit EK", + "version": "5" + }, + { + "authors": [ + "Kafeine", + "Will Metcalf", + "KahuSecurity" + ], + "description": "DealersChoice is a Flash Player Exploit platform triggered by RTF", + "galaxy_id": "370", + "id": "38805", + "meta": { + "refs": [ + "http://researchcenter.paloaltonetworks.com/2016/10/unit42-dealerschoice-sofacys-flash-player-exploit-platform/", + "http://blog.trendmicro.com/trendlabs-security-intelligence/pawn-storm-ramps-up-spear-phishing-before-zero-days-get-patched/" + ], + "status": [ + "Active" + ], + "synonyms": [ + "Sednit RTF EK" + ] + }, + "source": "MISP Project", + "tag_id": "3015", + "tag_name": "misp-galaxy:exploit-kit=\"DealersChoice\"", + "type": "exploit-kit", + "uuid": "454f4e78-bd7c-11e6-a4a6-cec0c932ce01", + "value": "DealersChoice", + "version": "5" + } + ], + "description": "Exploit-Kit is an enumeration of some exploitation kits used by adversaries. The list includes document, browser and router exploit kits.It's not meant to be totally exhaustive but aim at covering the most seen in the past 5 years", + "icon": "internet-explorer", + "id": "370", + "name": "Exploit-Kit", + "type": "exploit-kit", + "uuid": "6ab240ec-bd79-11e6-a4a6-cec0c932ce01", + "version": "3" + }, + { + "GalaxyCluster": [ + { + "authors": [ + "Alexandre Dulaunoy", + "Florian Roth", + "Timo Steffens", + "Christophe Vandeplas" + ], + "description": "backdoor", + "galaxy_id": "367", + "id": "46592", + "meta": { + "refs": [ + "https://www2.fireeye.com/rs/848-DID-242/images/APT28-Center-of-Storm-2017.pdf" + ], + "synonyms": [ + "Sednit", + "Seduploader", + "JHUHUGIT", + "Sofacy" + ], + "type": [ + "Backdoor" + ] + }, + "source": "MISP Project", + "tag_id": "2215", + "tag_name": "misp-galaxy:tool=\"GAMEFISH\"", + "type": "tool", + "uuid": "0d821b68-9d82-4c6d-86a6-1071a9e0f79f", + "value": "GAMEFISH", + "version": "45" + }, + { + "authors": [ + "Alexandre Dulaunoy", + "Florian Roth", + "Timo Steffens", + "Christophe Vandeplas" + ], + "description": "", + "galaxy_id": "367", + "id": "46670", + "meta": { + "synonyms": [ + "XTunnel" + ] + }, + "source": "MISP Project", + "tag_id": "1012", + "tag_name": "misp-galaxy:tool=\"X-Tunnel\"", + "type": "tool", + "uuid": "0d821b68-9d82-4c6d-86a6-1071a9e0f79f", + "value": "X-Tunnel", + "version": "45" + }, + { + "authors": [ + "Alexandre Dulaunoy", + "Florian Roth", + "Timo Steffens", + "Christophe Vandeplas" + ], + "description": "backdoor used by apt28\n\nSedreco serves as a spying backdoor; its functionalities can be extended with dynamically loaded plugins. It is made up of two distinct components: a dropper and the persistent payload installed by this dropper. We have not seen this component since April 2016.", + "galaxy_id": "367", + "id": "46591", + "meta": { + "possible_issues": [ + "Report tells that is could be Xagent alias (Java Rat)" + ], + "refs": [ + "https://www2.fireeye.com/rs/848-DID-242/images/APT28-Center-of-Storm-2017.pdf" + ], + "synonyms": [ + "Sedreco", + "AZZY", + "ADVSTORESHELL", + "NETUI" + ], + "type": [ + "Backdoor" + ] + }, + "source": "MISP Project", + "tag_id": "3011", + "tag_name": "misp-galaxy:tool=\"EVILTOSS\"", + "type": "tool", + "uuid": "0d821b68-9d82-4c6d-86a6-1071a9e0f79f", + "value": "EVILTOSS", + "version": "45" + }, + { + "authors": [ + "Alexandre Dulaunoy", + "Florian Roth", + "Timo Steffens", + "Christophe Vandeplas" + ], + "description": "This backdoor component is known to have a modular structure featuring various espionage functionalities, such as key-logging, screen grabbing and file exfiltration. This component is available for Osx, Windows, Linux and iOS operating systems.\n\nXagent is a modular backdoor with spying functionalities such as keystroke logging and file exfiltration. Xagent is the group\u2019s flagship backdoor and heavily used in their operations. Early versions for Linux and Windows were seen years ago, then in 2015 an iOS version came out. One year later, an Android version was discovered and finally, in the beginning of 2017, an Xagent sample for OS X was described.", + "galaxy_id": "367", + "id": "46669", + "meta": { + "refs": [ + "http://blog.trendmicro.com/trendlabs-security-intelligence/pawn-storm-update-ios-espionage-app-found/", + "https://app.box.com/s/l7n781ig6n8wlf1aff5hgwbh4qoi5jqq", + "https://www.welivesecurity.com/2017/12/21/sednit-update-fancy-bear-spent-year/" + ], + "synonyms": [ + "XAgent" + ], + "type": [ + "Backdoor" + ] + }, + "source": "MISP Project", + "tag_id": "1011", + "tag_name": "misp-galaxy:tool=\"X-Agent\"", + "type": "tool", + "uuid": "0d821b68-9d82-4c6d-86a6-1071a9e0f79f", + "value": "X-Agent", + "version": "45" + } + ], + "description": "Threat actors tools is an enumeration of tools used by adversaries. The list includes malware but also common software regularly used by the adversaries.", + "icon": "optin-monster", + "id": "367", + "name": "Tool", + "type": "tool", + "uuid": "9b8037f7-bc8f-4de1-a797-37266619bc0b", + "version": "2" + }, + { + "GalaxyCluster": [ + { + "authors": [ + "MITRE" + ], + "description": "JHUHUGIT is malware used by APT28. It is based on Carberp source code and serves as reconnaissance malware.[[Citation: Kaspersky Sofacy]][[Citation: F-Secure Sofacy 2015]][[Citation: ESET Sednit Part 1]][[Citation: FireEye APT28 January 2017]]\n\nAliases: JHUHUGIT, Seduploader, JKEYSKW, Sednit, GAMEFISH", + "galaxy_id": "365", + "id": "41618", + "meta": { + "refs": [ + "https://attack.mitre.org/wiki/Software/S0044", + "http://www.welivesecurity.com/wp-content/uploads/2016/10/eset-sednit-part1.pdf", + "https://www2.fireeye.com/rs/848-DID-242/images/APT28-Center-of-Storm-2017.pdf", + "https://labsblog.f-secure.com/2015/09/08/sofacy-recycles-carberp-and-metasploit-code/", + "https://securelist.com/blog/research/72924/sofacy-apt-hits-high-profile-targets-with-updated-toolset/" + ], + "synonyms": [ + "JHUHUGIT", + "Seduploader", + "JKEYSKW", + "Sednit", + "GAMEFISH" + ], + "uuid": [ + "8ae43c46-57ef-47d5-a77a-eebb35628db2" + ] + }, + "source": "https://github.com/mitre/cti", + "tag_id": "3008", + "tag_name": "misp-galaxy:mitre-malware=\"JHUHUGIT\"", + "type": "mitre-malware", + "uuid": "d752161c-78f6-11e7-a0ea-bfa79b407ce4", + "value": "JHUHUGIT", + "version": "4" + }, + { + "authors": [ + "MITRE" + ], + "description": "XTunnel a VPN-like network proxy tool that can relay traffic between a C2 server and a victim. It was first seen in May 2013 and reportedly used by APT28 during the compromise of the Democratic National Committee.[[Citation: Crowdstrike DNC June 2016]][[Citation: Invincea XTunnel]][[Citation: ESET Sednit Part 2]]\n\nAliases: XTunnel, X-Tunnel, XAPS", + "galaxy_id": "365", + "id": "41543", + "meta": { + "refs": [ + "https://attack.mitre.org/wiki/Software/S0117", + "http://www.welivesecurity.com/wp-content/uploads/2016/10/eset-sednit-part-2.pdf", + "https://www.invincea.com/2016/07/tunnel-of-gov-dnc-hack-and-the-russian-xtunnel/", + "https://www.crowdstrike.com/blog/bears-midst-intrusion-democratic-national-committee/" + ], + "synonyms": [ + "XTunnel", + "X-Tunnel", + "XAPS" + ], + "uuid": [ + "7343e208-7cab-45f2-a47b-41ba5e2f0fab" + ] + }, + "source": "https://github.com/mitre/cti", + "tag_id": "3009", + "tag_name": "misp-galaxy:mitre-malware=\"XTunnel\"", + "type": "mitre-malware", + "uuid": "d752161c-78f6-11e7-a0ea-bfa79b407ce4", + "value": "XTunnel", + "version": "4" + }, + { + "authors": [ + "MITRE" + ], + "description": "ADVSTORESHELL is a spying backdoor that has been used by APT28 from at least 2012 to 2016. It is generally used for long-term espionage and is deployed on targets deemed interesting after a reconnaissance phase.[[Citation: Kaspersky Sofacy]][[Citation: ESET Sednit Part 2]]\n\nAliases: ADVSTORESHELL, NETUI, EVILTOSS, AZZY, Sedreco", + "galaxy_id": "365", + "id": "41582", + "meta": { + "refs": [ + "https://attack.mitre.org/wiki/Software/S0045", + "http://www.welivesecurity.com/wp-content/uploads/2016/10/eset-sednit-part-2.pdf", + "https://securelist.com/blog/research/72924/sofacy-apt-hits-high-profile-targets-with-updated-toolset/" + ], + "synonyms": [ + "ADVSTORESHELL", + "NETUI", + "EVILTOSS", + "AZZY", + "Sedreco" + ], + "uuid": [ + "fb575479-14ef-41e9-bfab-0b7cf10bec73" + ] + }, + "source": "https://github.com/mitre/cti", + "tag_id": "3010", + "tag_name": "misp-galaxy:mitre-malware=\"ADVSTORESHELL\"", + "type": "mitre-malware", + "uuid": "d752161c-78f6-11e7-a0ea-bfa79b407ce4", + "value": "ADVSTORESHELL", + "version": "4" + }, + { + "authors": [ + "MITRE" + ], + "description": "USBStealer is malware that has used by APT28 since at least 2005 to extract information from air-gapped networks. It does not have the capability to communicate over the Internet and has been used in conjunction with ADVSTORESHELL.[[Citation: ESET Sednit USBStealer 2014]][[Citation: Kaspersky Sofacy]]\n\nAliases: USBStealer, USB Stealer, Win32/USBStealer", + "galaxy_id": "365", + "id": "41549", + "meta": { + "refs": [ + "https://attack.mitre.org/wiki/Software/S0136", + "http://www.welivesecurity.com/2014/11/11/sednit-espionage-group-attacking-air-gapped-networks/", + "https://securelist.com/blog/research/72924/sofacy-apt-hits-high-profile-targets-with-updated-toolset/" + ], + "synonyms": [ + "USBStealer", + "USB Stealer", + "Win32/USBStealer" + ], + "uuid": [ + "af2ad3b7-ab6a-4807-91fd-51bcaff9acbb" + ] + }, + "source": "https://github.com/mitre/cti", + "tag_id": "3012", + "tag_name": "misp-galaxy:mitre-malware=\"USBStealer\"", + "type": "mitre-malware", + "uuid": "d752161c-78f6-11e7-a0ea-bfa79b407ce4", + "value": "USBStealer", + "version": "4" + }, + { + "authors": [ + "MITRE" + ], + "description": "is a trojan that has been used by APT28 on OS X and appears to be a port of their standard CHOPSTICK or XAgent trojan.[[Citation: XAgentOSX]]", + "galaxy_id": "365", + "id": "41551", + "meta": { + "refs": [ + "https://attack.mitre.org/wiki/Software/S0161", + "https://researchcenter.paloaltonetworks.com/2017/02/unit42-xagentosx-sofacys-xagent-macos-tool/" + ], + "uuid": [ + "5930509b-7793-4db9-bdfc-4edda7709d0d" + ] + }, + "source": "https://github.com/mitre/cti", + "tag_id": "3013", + "tag_name": "misp-galaxy:mitre-malware=\"XAgentOSX\"", + "type": "mitre-malware", + "uuid": "d752161c-78f6-11e7-a0ea-bfa79b407ce4", + "value": "XAgentOSX", + "version": "4" + }, + { + "authors": [ + "MITRE" + ], + "description": "CHOPSTICK is malware family of modular backdoors used by APT28. It has been used from at least November 2012 to August 2016 and is usually dropped on victims as second-stage malware, though it has been used as first-stage malware in several cases.[[Citation: FireEye APT28]][[Citation: ESET Sednit Part 2]][[Citation: FireEye APT28 January 2017]]\n\nAliases: CHOPSTICK, SPLM, Xagent, X-Agent, webhp", + "galaxy_id": "365", + "id": "41559", + "meta": { + "refs": [ + "https://attack.mitre.org/wiki/Software/S0023", + "https://www2.fireeye.com/rs/848-DID-242/images/APT28-Center-of-Storm-2017.pdf", + "http://www.welivesecurity.com/wp-content/uploads/2016/10/eset-sednit-part-2.pdf", + "https://www.fireeye.com/content/dam/fireeye-www/global/en/current-threats/pdfs/rpt-apt28.pdf" + ], + "synonyms": [ + "CHOPSTICK", + "SPLM", + "Xagent", + "X-Agent", + "webhp" + ], + "uuid": [ + "ccd61dfc-b03f-4689-8c18-7c97eab08472" + ] + }, + "source": "https://github.com/mitre/cti", + "tag_id": "3014", + "tag_name": "misp-galaxy:mitre-malware=\"CHOPSTICK\"", + "type": "mitre-malware", + "uuid": "d752161c-78f6-11e7-a0ea-bfa79b407ce4", + "value": "CHOPSTICK", + "version": "4" + }, + { + "authors": [ + "MITRE" + ], + "description": "Downdelph is a first-stage downloader written in Delphi that has been used by APT28 in rare instances between 2013 and 2015.[[Citation: ESET Sednit Part 3]]\n\nAliases: Downdelph, Delphacy", + "galaxy_id": "365", + "id": "41504", + "meta": { + "refs": [ + "https://attack.mitre.org/wiki/Software/S0134", + "http://www.welivesecurity.com/wp-content/uploads/2016/10/eset-sednit-part3.pdf" + ], + "synonyms": [ + "Downdelph", + "Delphacy" + ], + "uuid": [ + "08d20cd2-f084-45ee-8558-fa6ef5a18519" + ] + }, + "source": "https://github.com/mitre/cti", + "tag_id": "3016", + "tag_name": "misp-galaxy:mitre-malware=\"Downdelph\"", + "type": "mitre-malware", + "uuid": "d752161c-78f6-11e7-a0ea-bfa79b407ce4", + "value": "Downdelph", + "version": "4" + } + ], + "description": "Name of ATT&CK software", + "icon": "optin-monster", + "id": "365", + "name": "Malware", + "type": "mitre-malware", + "uuid": "d752161c-78f6-11e7-a0ea-bfa79b407ce4", + "version": "4" + } + ], + "Object": [ + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188944", + "object_id": "1555", + "object_relation": "filename", + "sharing_group_id": "0", + "timestamp": "1513936310", + "to_ids": true, + "type": "filename", + "uuid": "5a3cd5b6-2850-435f-bd0d-4c62950d210f", + "value": "Bulletin.doc" + }, + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188945", + "object_id": "1555", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513936310", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd5b6-78a8-4e47-8333-4c62950d210f", + "value": "68064fc152e23d56e541714af52651cb4ba81aaf" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188946", + "object_id": "1555", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513936310", + "to_ids": false, + "type": "text", + "uuid": "5a3cd5b6-23d8-43ba-8518-4c62950d210f", + "value": "Malicious" + } + ], + "comment": "Win32/Sednit.AX", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1555", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513936310", + "uuid": "5a3cd5b6-9568-4342-b2ab-4c62950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188947", + "object_id": "1556", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513936388", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd604-748c-4fc0-88bf-c170950d210f", + "value": "f3805382ae2e23ff1147301d131a06e00e4ff75f" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188948", + "object_id": "1556", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513936388", + "to_ids": false, + "type": "text", + "uuid": "5a3cd604-6668-4469-a1c0-c170950d210f", + "value": "Malicious" + } + ], + "comment": "Win32/Exploit.CVE-2016-4117.A", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1556", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513936388", + "uuid": "5a3cd604-e11c-4de5-bbbf-c170950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188949", + "object_id": "1557", + "object_relation": "filename", + "sharing_group_id": "0", + "timestamp": "1513936531", + "to_ids": true, + "type": "filename", + "uuid": "5a3cd693-dc40-445d-a4d7-4ae0950d210f", + "value": "OC_PSO_2017.doc" + }, + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188950", + "object_id": "1557", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513936531", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd693-8ffc-4d95-b522-4e84950d210f", + "value": "512bdfe937314ac3f195c462c395feeb36932971" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188951", + "object_id": "1557", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513936531", + "to_ids": false, + "type": "text", + "uuid": "5a3cd693-a8f0-4aea-a834-4097950d210f", + "value": "Malicious" + } + ], + "comment": "Win32/Exploit.Agent.NUB", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1557", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513936531", + "uuid": "5a3cd693-fd9c-4fcf-b69a-439c950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188952", + "object_id": "1558", + "object_relation": "filename", + "sharing_group_id": "0", + "timestamp": "1513936578", + "to_ids": true, + "type": "filename", + "uuid": "5a3cd6c2-d31c-40cc-bcc1-4458950d210f", + "value": "NASAMS.doc" + }, + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188953", + "object_id": "1558", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513936578", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd6c2-6a54-4b4c-8748-4c84950d210f", + "value": "30b3e8c0f3f3cf200daa21c267ffab3cad64e68b" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188954", + "object_id": "1558", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513936578", + "to_ids": false, + "type": "text", + "uuid": "5a3cd6c2-1c68-45de-8325-464a950d210f", + "value": "Malicious" + } + ], + "comment": "Win32/Exploit.Agent.NTR", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1558", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513936578", + "uuid": "5a3cd6c2-d290-4787-910f-4e6d950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188955", + "object_id": "1559", + "object_relation": "filename", + "sharing_group_id": "0", + "timestamp": "1513936718", + "to_ids": true, + "type": "filename", + "uuid": "5a3cd74e-584c-45b9-8557-486d950d210f", + "value": "Programm_Details.doc" + }, + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188956", + "object_id": "1559", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513936718", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd74e-f334-4e6b-b37f-462f950d210f", + "value": "4173b29a251cd9c1cab135f67cb60acab4ace0c5" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188957", + "object_id": "1559", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513936718", + "to_ids": false, + "type": "text", + "uuid": "5a3cd74e-5900-4fbf-85c6-4c81950d210f", + "value": "Malicious" + } + ], + "comment": "Win32/Exploit.Agent.NTO", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1559", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513936718", + "uuid": "5a3cd74e-1504-40ff-9a28-4501950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188958", + "object_id": "1560", + "object_relation": "filename", + "sharing_group_id": "0", + "timestamp": "1513936757", + "to_ids": true, + "type": "filename", + "uuid": "5a3cd775-e8f4-465a-aca2-4c5a950d210f", + "value": "Operation_in_Mosul.rtf" + }, + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188959", + "object_id": "1560", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513936757", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd775-1190-4db7-961a-4c5a950d210f", + "value": "12a37cfdd3f3671074dd5b0f354269cec028fb52" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188960", + "object_id": "1560", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513936757", + "to_ids": false, + "type": "text", + "uuid": "5a3cd775-fa5c-4453-bcb0-4c5a950d210f", + "value": "Malicious" + } + ], + "comment": "Win32/Exploit.Agent.NTR", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1560", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513936757", + "uuid": "5a3cd775-e4cc-44bb-89b6-4c5a950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188961", + "object_id": "1561", + "object_relation": "filename", + "sharing_group_id": "0", + "timestamp": "1513936943", + "to_ids": true, + "type": "filename", + "uuid": "5a3cd82f-b918-4520-ba8b-5165950d210f", + "value": "ARM-NATO_ENGLISH_30_NOV_2016.doc" + }, + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188962", + "object_id": "1561", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513936943", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd82f-cae4-4209-9338-5165950d210f", + "value": "15201766bd964b7c405aeb11db81457220c31e46" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188963", + "object_id": "1561", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513936943", + "to_ids": false, + "type": "text", + "uuid": "5a3cd82f-d91c-43af-8262-5165950d210f", + "value": "Malicious" + } + ], + "comment": "SWF/Agent.L", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1561", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513936943", + "uuid": "5a3cd82f-2788-4561-bbeb-5165950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188964", + "object_id": "1562", + "object_relation": "filename", + "sharing_group_id": "0", + "timestamp": "1513936967", + "to_ids": true, + "type": "filename", + "uuid": "5a3cd847-0aa0-4b5c-aa30-5165950d210f", + "value": "Olympic-Agenda-2020-20-20-Recommendations.doc" + }, + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188965", + "object_id": "1562", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513936967", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd847-593c-4985-8756-5165950d210f", + "value": "8078e411fbe33864dfd8f87ad5105cc1fd26d62e" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188966", + "object_id": "1562", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513936967", + "to_ids": false, + "type": "text", + "uuid": "5a3cd847-1324-4fad-af60-5165950d210f", + "value": "Malicious" + } + ], + "comment": "Win32/Exploit.Agent.BL", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1562", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513936967", + "uuid": "5a3cd847-b5a0-42f7-ac4b-5165950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188967", + "object_id": "1563", + "object_relation": "filename", + "sharing_group_id": "0", + "timestamp": "1513936993", + "to_ids": true, + "type": "filename", + "uuid": "5a3cd861-9350-40c1-ac29-4771950d210f", + "value": "Merry_Christmas!.docx" + }, + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188968", + "object_id": "1563", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513936993", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd861-18ac-4cf0-b96f-4986950d210f", + "value": "33447383379ca99083442b852589111296f0c603" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188969", + "object_id": "1563", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513936993", + "to_ids": false, + "type": "text", + "uuid": "5a3cd861-cfbc-4096-baae-40e2950d210f", + "value": "Malicious" + } + ], + "comment": "Win32/Exploit.Agent.NUG", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1563", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513936993", + "uuid": "5a3cd861-65c0-4b69-9429-4f37950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188970", + "object_id": "1564", + "object_relation": "filename", + "sharing_group_id": "0", + "timestamp": "1513937021", + "to_ids": true, + "type": "filename", + "uuid": "5a3cd87d-fa9c-41aa-897f-49a5950d210f", + "value": "Trump\u2019s_Attack_on_Syria_English.docx" + }, + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188971", + "object_id": "1564", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937021", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd87d-c630-4487-8336-4615950d210f", + "value": "d5235d136cfcadbef431eea7253d80bde414db9d" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188972", + "object_id": "1564", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937021", + "to_ids": false, + "type": "text", + "uuid": "5a3cd87d-8c98-4660-9026-44de950d210f", + "value": "Malicious" + } + ], + "comment": "Win32/Exploit.Agent.NWZ", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1564", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513937021", + "uuid": "5a3cd87d-f514-4071-a5f7-4ec2950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188973", + "object_id": "1565", + "object_relation": "filename", + "sharing_group_id": "0", + "timestamp": "1513937047", + "to_ids": true, + "type": "filename", + "uuid": "5a3cd897-4cc0-48b0-bb2c-461f950d210f", + "value": "Hotel_Reservation_Form.doc" + }, + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188974", + "object_id": "1565", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937047", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd897-fa64-466c-9421-49c5950d210f", + "value": "f293a2bfb728060c54efeeb03c5323893b5c80df" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188975", + "object_id": "1565", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937047", + "to_ids": false, + "type": "text", + "uuid": "5a3cd897-f020-44cf-8dfc-4225950d210f", + "value": "Malicious" + } + ], + "comment": "Win32/Sednit.BN", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1565", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513937046", + "uuid": "5a3cd896-f6cc-4e52-bcb2-442c950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188976", + "object_id": "1566", + "object_relation": "filename", + "sharing_group_id": "0", + "timestamp": "1513937070", + "to_ids": true, + "type": "filename", + "uuid": "5a3cd8ae-7194-48fd-810e-4c5a950d210f", + "value": "SB_Doc_2017-3_Implementation_of_Key_Taskings_and_Next_Steps.doc" + }, + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188977", + "object_id": "1566", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937071", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd8af-f39c-443c-bcf1-4c5a950d210f", + "value": "bb10ed5d59672fbc6178e35d0feac0562513e9f0" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188978", + "object_id": "1566", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937071", + "to_ids": false, + "type": "text", + "uuid": "5a3cd8af-b3ec-478a-b585-4c5a950d210f", + "value": "Malicious" + } + ], + "comment": "Win32/Sednit.BN", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1566", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513937070", + "uuid": "5a3cd8ae-54d0-46bb-adbb-4c5a950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188979", + "object_id": "1567", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937083", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd8bb-74d8-4d19-ae08-4043950d210f", + "value": "4873bafe44cff06845faa0ce7c270c4ce3c9f7b9" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188980", + "object_id": "1567", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937083", + "to_ids": false, + "type": "text", + "uuid": "5a3cd8bb-77bc-4cc4-887f-429d950d210f", + "value": "Malicious" + } + ], + "comment": "", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1567", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513937083", + "uuid": "5a3cd8bb-a704-4f1d-a235-444e950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188981", + "object_id": "1568", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937097", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd8c9-4d2c-4145-a637-4f13950d210f", + "value": "169c8f3e3d22e192c108bc95164d362ce5437465" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188982", + "object_id": "1568", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937097", + "to_ids": false, + "type": "text", + "uuid": "5a3cd8c9-7ff0-42f7-ae80-4eb6950d210f", + "value": "Malicious" + } + ], + "comment": "", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1568", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513937097", + "uuid": "5a3cd8c9-6568-406a-853c-4862950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188983", + "object_id": "1569", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937116", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd8dc-48c0-4ea0-a67d-4734950d210f", + "value": "cc7607015cd7a1a4452acd3d87adabdd7e005bd7" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188984", + "object_id": "1569", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937116", + "to_ids": false, + "type": "text", + "uuid": "5a3cd8dc-9ed8-4a4d-9ceb-4daa950d210f", + "value": "Malicious" + } + ], + "comment": "Win32/Sednit.BN", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1569", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513937115", + "uuid": "5a3cd8db-2838-4466-a986-4afb950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188985", + "object_id": "1570", + "object_relation": "filename", + "sharing_group_id": "0", + "timestamp": "1513937147", + "to_ids": true, + "type": "filename", + "uuid": "5a3cd8fb-1efc-4059-ae7a-42f5950d210f", + "value": "Caucasian_Eagle_ENG.docx" + }, + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188986", + "object_id": "1570", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937147", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd8fb-9cec-4a30-8b2f-4441950d210f", + "value": "5d2c7d87995cc5b8184baba2c7a1900a48b2f42d" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188987", + "object_id": "1570", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937147", + "to_ids": false, + "type": "text", + "uuid": "5a3cd8fb-e52c-489b-8da5-43d1950d210f", + "value": "Malicious" + } + ], + "comment": "Win32/Exploit.Agent.NTM", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1570", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513937147", + "uuid": "5a3cd8fb-cd14-4b00-9710-430c950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188988", + "object_id": "1571", + "object_relation": "filename", + "sharing_group_id": "0", + "timestamp": "1513937166", + "to_ids": true, + "type": "filename", + "uuid": "5a3cd90e-5eb4-4069-b160-5276950d210f", + "value": "World War3.docx" + }, + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188989", + "object_id": "1571", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937166", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd90e-6d2c-4ffc-a699-5276950d210f", + "value": "7aada8bcc0d1ab8ffb1f0fae4757789c6f5546a3" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188990", + "object_id": "1571", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937166", + "to_ids": false, + "type": "text", + "uuid": "5a3cd90e-28e8-410e-8033-5276950d210f", + "value": "Malicious" + } + ], + "comment": "SWF/Exploit.CVE-2017-11292.A", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1571", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513937166", + "uuid": "5a3cd90e-538c-4b7e-95dc-5276950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188991", + "object_id": "1572", + "object_relation": "filename", + "sharing_group_id": "0", + "timestamp": "1513937191", + "to_ids": true, + "type": "filename", + "uuid": "5a3cd927-e810-4d22-a0e4-4057950d210f", + "value": "SaberGuardian2017.docx" + }, + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188992", + "object_id": "1572", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937191", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd927-f284-43b9-83d1-473b950d210f", + "value": "68c2809560c7623d2307d8797691abf3eafe319a" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188993", + "object_id": "1572", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937191", + "to_ids": false, + "type": "text", + "uuid": "5a3cd927-b844-49f2-a1a9-4c85950d210f", + "value": "Malicious" + } + ], + "comment": "VBA/DDE.E", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1572", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513937191", + "uuid": "5a3cd927-e410-489c-abfc-4b63950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188994", + "object_id": "1573", + "object_relation": "filename", + "sharing_group_id": "0", + "timestamp": "1513937212", + "to_ids": true, + "type": "filename", + "uuid": "5a3cd93c-2438-4dda-823e-463d950d210f", + "value": "IsisAttackInNewYork.docx" + }, + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188995", + "object_id": "1573", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937212", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd93c-1ef0-4d81-9476-4655950d210f", + "value": "1c6c700ceebfbe799e115582665105caa03c5c9e" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188996", + "object_id": "1573", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937212", + "to_ids": false, + "type": "text", + "uuid": "5a3cd93c-949c-40ac-9094-4a4a950d210f", + "value": "Malicious" + } + ], + "comment": "VBA/DDE.L", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1573", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513937212", + "uuid": "5a3cd93c-716c-4918-a00f-4671950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188997", + "object_id": "1574", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937559", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cda97-7e58-4642-aaf5-c5ed950d210f", + "value": "6f0fc0ebba3e4c8b26a69cdf519edf8d1aa2f4bb" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188998", + "object_id": "1574", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937559", + "to_ids": false, + "type": "text", + "uuid": "5a3cda97-6020-423d-9d23-c5ed950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Attribute": { + "category": "Network activity", + "distribution": "5", + "sharing_group_id": "0", + "to_ids": true, + "type": "domain", + "uuid": "5a3c3045-ab0c-4d38-8efe-459002de0b81", + "value": "movieultimate.com" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "159", + "object_id": "1574", + "object_uuid": "5a3cda96-85c4-45a1-82ea-c5ed950d210f", + "referenced_id": "1188759", + "referenced_type": "0", + "referenced_uuid": "5a3c3045-ab0c-4d38-8efe-459002de0b81", + "relationship_type": "communicates-with", + "timestamp": "1513937826", + "uuid": "5a3cdba2-2fdc-4f9a-a4eb-4dae950d210f" + } + ], + "comment": "Win64/Sednit.Z", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1574", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513937826", + "uuid": "5a3cda96-85c4-45a1-82ea-c5ed950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188999", + "object_id": "1575", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937864", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cdbc8-0aac-4d8a-8c1f-4c5a950d210f", + "value": "e19f753e514f6adec8f81bcdefb9117979e69627" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189000", + "object_id": "1575", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937864", + "to_ids": false, + "type": "text", + "uuid": "5a3cdbc8-e204-4606-b9ea-4c5a950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Attribute": { + "category": "Network activity", + "distribution": "5", + "sharing_group_id": "0", + "to_ids": true, + "type": "domain", + "uuid": "5a3c3045-61dc-495c-ae8a-471e02de0b81", + "value": "meteost.com" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "160", + "object_id": "1575", + "object_uuid": "5a3cdbc7-dbec-4b8c-8ba3-4c5a950d210f", + "referenced_id": "1188760", + "referenced_type": "0", + "referenced_uuid": "5a3c3045-61dc-495c-ae8a-471e02de0b81", + "relationship_type": "communicates-with", + "timestamp": "1513938091", + "uuid": "5a3cdcab-8200-4c65-868e-42a9950d210f" + } + ], + "comment": "Win64/Sednit.Z", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1575", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513938091", + "uuid": "5a3cdbc7-dbec-4b8c-8ba3-4c5a950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189001", + "object_id": "1576", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937910", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cdbf6-eca0-4c09-9bd0-4c59950d210f", + "value": "961468ddd3d0fa25beb8210c81ba620f9170ed30" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189002", + "object_id": "1576", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937910", + "to_ids": false, + "type": "text", + "uuid": "5a3cdbf6-acd8-4a36-a028-4c59950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Attribute": { + "category": "Network activity", + "distribution": "5", + "sharing_group_id": "0", + "to_ids": true, + "type": "domain", + "uuid": "5a3c3045-e354-4978-a6b4-49ad02de0b81", + "value": "faststoragefiles.org" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "164", + "object_id": "1576", + "object_uuid": "5a3cdbf6-f814-491f-9f93-4c59950d210f", + "referenced_id": "1188761", + "referenced_type": "0", + "referenced_uuid": "5a3c3045-e354-4978-a6b4-49ad02de0b81", + "relationship_type": "communicates-with", + "timestamp": "1513938210", + "uuid": "5a3cdd22-b7d8-4754-a108-4742950d210f" + } + ], + "comment": "Win32/Sednit.BO", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1576", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513938210", + "uuid": "5a3cdbf6-f814-491f-9f93-4c59950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189003", + "object_id": "1577", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937929", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cdc09-b428-4c0b-9969-c5ed950d210f", + "value": "a0719b50265505c8432616c0a4e14ed206981e95" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189004", + "object_id": "1577", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937929", + "to_ids": false, + "type": "text", + "uuid": "5a3cdc09-05d8-4356-ba52-c5ed950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Attribute": { + "category": "Network activity", + "distribution": "5", + "sharing_group_id": "0", + "to_ids": true, + "type": "domain", + "uuid": "5a3c3045-968c-4572-9f64-491502de0b81", + "value": "nethostnet.com" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "162", + "object_id": "1577", + "object_uuid": "5a3cdc09-6fbc-4ca1-bfaa-c5ed950d210f", + "referenced_id": "1188762", + "referenced_type": "0", + "referenced_uuid": "5a3c3045-968c-4572-9f64-491502de0b81", + "relationship_type": "communicates-with", + "timestamp": "1513938169", + "uuid": "5a3cdcf9-d5a4-4c8e-a201-45b1950d210f" + } + ], + "comment": "Win32/Sednit.BO", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1577", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513938169", + "uuid": "5a3cdc09-6fbc-4ca1-bfaa-c5ed950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189005", + "object_id": "1578", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937953", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cdc21-a170-4637-b139-4812950d210f", + "value": "2cf6436b99d11d9d1e0c488af518e35162ecbc9c" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189006", + "object_id": "1578", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937953", + "to_ids": false, + "type": "text", + "uuid": "5a3cdc21-3274-4800-9e91-41e2950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Attribute": { + "category": "Network activity", + "distribution": "5", + "sharing_group_id": "0", + "to_ids": true, + "type": "domain", + "uuid": "5a3c3045-e354-4978-a6b4-49ad02de0b81", + "value": "faststoragefiles.org" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "165", + "object_id": "1578", + "object_uuid": "5a3cdc21-856c-48bd-a757-4f4b950d210f", + "referenced_id": "1188761", + "referenced_type": "0", + "referenced_uuid": "5a3c3045-e354-4978-a6b4-49ad02de0b81", + "relationship_type": "communicates-with", + "timestamp": "1513938226", + "uuid": "5a3cdd32-3044-4895-8f18-4d06950d210f" + } + ], + "comment": "Win64/Sednit.Y", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1578", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513938226", + "uuid": "5a3cdc21-856c-48bd-a757-4f4b950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189007", + "object_id": "1579", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937975", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cdc37-cee0-43d0-9e20-4db6950d210f", + "value": "fec29b4f4dccc59770c65c128dfe4564d7c13d33" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189008", + "object_id": "1579", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937976", + "to_ids": false, + "type": "text", + "uuid": "5a3cdc38-ac24-44be-a1ed-4935950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Attribute": { + "category": "Network activity", + "distribution": "5", + "sharing_group_id": "0", + "to_ids": true, + "type": "domain", + "uuid": "5a3c3045-eb44-433f-a13a-44b902de0b81", + "value": "fsportal.net" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "163", + "object_id": "1579", + "object_uuid": "5a3cdc37-89e8-4a2d-823a-4af8950d210f", + "referenced_id": "1188763", + "referenced_type": "0", + "referenced_uuid": "5a3c3045-eb44-433f-a13a-44b902de0b81", + "relationship_type": "communicates-with", + "timestamp": "1513938189", + "uuid": "5a3cdd0d-d990-42ba-830d-5156950d210f" + } + ], + "comment": "Win64/Sednit.Y", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1579", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513938190", + "uuid": "5a3cdc37-89e8-4a2d-823a-4af8950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189009", + "object_id": "1580", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937992", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cdc48-c74c-4b6e-8202-5156950d210f", + "value": "57d7f3d31c491f8aef4665ca4dd905c3c8a98795" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189010", + "object_id": "1580", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937992", + "to_ids": false, + "type": "text", + "uuid": "5a3cdc48-55dc-420e-9b5d-5156950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Attribute": { + "category": "Network activity", + "distribution": "5", + "sharing_group_id": "0", + "to_ids": true, + "type": "domain", + "uuid": "5a3c3045-6a88-479d-b799-4d3d02de0b81", + "value": "fastdataexchange.org" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "161", + "object_id": "1580", + "object_uuid": "5a3cdc48-b9a0-4775-a03f-5156950d210f", + "referenced_id": "1188764", + "referenced_type": "0", + "referenced_uuid": "5a3c3045-6a88-479d-b799-4d3d02de0b81", + "relationship_type": "communicates-with", + "timestamp": "1513938129", + "uuid": "5a3cdcd1-c6cc-43d8-a2f4-4681950d210f" + } + ], + "comment": "Win64/Sednit.Z", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1580", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513938129", + "uuid": "5a3cdc48-b9a0-4775-a03f-5156950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189011", + "object_id": "1581", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513938011", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cdc5b-54a8-4e60-bc67-4c5a950d210f", + "value": "a3bf5b5cf5a5ef438a198a6f61f7225c0a4a7138" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189012", + "object_id": "1581", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513938011", + "to_ids": false, + "type": "text", + "uuid": "5a3cdc5b-b390-4183-aec7-4c5a950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Attribute": { + "category": "Network activity", + "distribution": "5", + "sharing_group_id": "0", + "to_ids": true, + "type": "domain", + "uuid": "5a3c3045-7480-4831-a5c4-48c802de0b81", + "value": "newfilmts.com" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "168", + "object_id": "1581", + "object_uuid": "5a3cdc5a-8760-4efa-949a-4c5a950d210f", + "referenced_id": "1188765", + "referenced_type": "0", + "referenced_uuid": "5a3c3045-7480-4831-a5c4-48c802de0b81", + "relationship_type": "communicates-with", + "timestamp": "1513938280", + "uuid": "5a3cdd68-7968-40d1-a0a9-5156950d210f" + } + ], + "comment": "Win32/Sednit.BO", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1581", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513938280", + "uuid": "5a3cdc5a-8760-4efa-949a-4c5a950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189013", + "object_id": "1582", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513938034", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cdc72-ba30-4ecd-9d21-4654950d210f", + "value": "1958e722afd0dba266576922abc98aa505cf5f9a" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189014", + "object_id": "1582", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513938034", + "to_ids": false, + "type": "text", + "uuid": "5a3cdc72-0804-42c4-acfa-4ac5950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Attribute": { + "category": "Network activity", + "distribution": "5", + "sharing_group_id": "0", + "to_ids": true, + "type": "domain", + "uuid": "5a3c3045-7480-4831-a5c4-48c802de0b81", + "value": "newfilmts.com" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "167", + "object_id": "1582", + "object_uuid": "5a3cdc72-1538-4c66-af46-427b950d210f", + "referenced_id": "1188765", + "referenced_type": "0", + "referenced_uuid": "5a3c3045-7480-4831-a5c4-48c802de0b81", + "relationship_type": "communicates-with", + "timestamp": "1513938264", + "uuid": "5a3cdd58-9800-4bae-837c-4f20950d210f" + } + ], + "comment": "Win32/Sednit.BO", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1582", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513938264", + "uuid": "5a3cdc72-1538-4c66-af46-427b950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189015", + "object_id": "1583", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513939882", + "to_ids": true, + "type": "sha1", + "uuid": "5a3ce3aa-e104-481e-a7f4-4bc1950d210f", + "value": "9f6bed7d7f4728490117cbc85819c2e6c494251b" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189016", + "object_id": "1583", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513939882", + "to_ids": false, + "type": "text", + "uuid": "5a3ce3aa-74fc-48c7-af40-4c6a950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Object": { + "distribution": "5", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "uuid": "5a3ce58a-3198-4cb8-9d51-44e5950d210f" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "173", + "object_id": "1583", + "object_uuid": "5a3ce3a9-f070-4403-a1f6-4b8c950d210f", + "referenced_id": "1592", + "referenced_type": "1", + "referenced_uuid": "5a3ce58a-3198-4cb8-9d51-44e5950d210f", + "relationship_type": "communicates-with", + "timestamp": "1513947459", + "uuid": "5a3d0143-c300-4118-8afe-4a2d950d210f" + } + ], + "comment": "Win32/Sednit.AX\t", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1583", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513948642", + "uuid": "5a3ce3a9-f070-4403-a1f6-4b8c950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189017", + "object_id": "1584", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513939907", + "to_ids": true, + "type": "sha1", + "uuid": "5a3ce3c3-6d9c-48f4-93db-4a61950d210f", + "value": "4bc722a9b0492a50bd86a1341f02c74c0d773db7" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189018", + "object_id": "1584", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513939907", + "to_ids": false, + "type": "text", + "uuid": "5a3ce3c3-c38c-4e30-a904-4c8f950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Object": { + "distribution": "5", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "uuid": "5a3ce6ae-98d8-4270-b88f-47f2950d210f" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "188", + "object_id": "1584", + "object_uuid": "5a3ce3c3-34b4-4e1f-b238-4399950d210f", + "referenced_id": "1603", + "referenced_type": "1", + "referenced_uuid": "5a3ce6ae-98d8-4270-b88f-47f2950d210f", + "relationship_type": "communicates-with", + "timestamp": "1513948518", + "uuid": "5a3d0566-34fc-4a62-b2a5-4f91950d210f" + } + ], + "comment": "Win32/Sednit.BS", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1584", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513948535", + "uuid": "5a3ce3c3-34b4-4e1f-b238-4399950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189019", + "object_id": "1585", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513939924", + "to_ids": true, + "type": "sha1", + "uuid": "5a3ce3d4-9168-4e23-8b64-485a950d210f", + "value": "ab354807e687993fbeb1b325eb6e4ab38d428a1e" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189020", + "object_id": "1585", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513939924", + "to_ids": false, + "type": "text", + "uuid": "5a3ce3d4-27e0-4366-943f-4b9a950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Object": { + "distribution": "5", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "uuid": "5a3ce6a1-3f1c-4d5d-bac7-406d950d210f" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "189", + "object_id": "1585", + "object_uuid": "5a3ce3d4-07bc-4af3-90fc-4798950d210f", + "referenced_id": "1602", + "referenced_type": "1", + "referenced_uuid": "5a3ce6a1-3f1c-4d5d-bac7-406d950d210f", + "relationship_type": "communicates-with", + "timestamp": "1513948528", + "uuid": "5a3d0570-a86c-4264-a43a-4125950d210f" + } + ], + "comment": "Win32/Sednit.BS", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1585", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513948597", + "uuid": "5a3ce3d4-07bc-4af3-90fc-4798950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189021", + "object_id": "1586", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513939946", + "to_ids": true, + "type": "sha1", + "uuid": "5a3ce3ea-8dbc-4cf4-997f-448b950d210f", + "value": "9c47ca3883196b3a84d67676a804ff50e22b0a9f" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189022", + "object_id": "1586", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513939946", + "to_ids": false, + "type": "text", + "uuid": "5a3ce3ea-e714-444e-ad9b-40b0950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Object": { + "distribution": "5", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "uuid": "5a3ce68d-1940-4ea6-becd-44fe950d210f" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "190", + "object_id": "1586", + "object_uuid": "5a3ce3ea-580c-477c-9b73-4e57950d210f", + "referenced_id": "1601", + "referenced_type": "1", + "referenced_uuid": "5a3ce68d-1940-4ea6-becd-44fe950d210f", + "relationship_type": "communicates-with", + "timestamp": "1513948614", + "uuid": "5a3d05c6-0618-4520-9549-48a0950d210f" + } + ], + "comment": "Win32/Sednit.BR", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1586", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513948626", + "uuid": "5a3ce3ea-580c-477c-9b73-4e57950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189023", + "object_id": "1587", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513939972", + "to_ids": true, + "type": "sha1", + "uuid": "5a3ce404-7bfc-4316-bd32-55ea950d210f", + "value": "8a68f26d01372114f660e32ac4c9117e5d0577f1" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189024", + "object_id": "1587", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513939972", + "to_ids": false, + "type": "text", + "uuid": "5a3ce404-7224-4525-922a-55ea950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Object": { + "distribution": "5", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "uuid": "5a3ce680-90d4-478d-95db-48a6950d210f" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "182", + "object_id": "1587", + "object_uuid": "5a3ce404-efc0-4f15-864e-55ea950d210f", + "referenced_id": "1600", + "referenced_type": "1", + "referenced_uuid": "5a3ce680-90d4-478d-95db-48a6950d210f", + "relationship_type": "communicates-with", + "timestamp": "1513948044", + "uuid": "5a3d038c-1cc8-4d9c-87ab-c5ed950d210f" + } + ], + "comment": "Win32/Sednit.BN", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1587", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513948073", + "uuid": "5a3ce404-efc0-4f15-864e-55ea950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189025", + "object_id": "1588", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513939991", + "to_ids": true, + "type": "sha1", + "uuid": "5a3ce417-62a4-4d46-9a87-55ea950d210f", + "value": "476fc1d31722ac26b46154cbf0c631d60268b28a" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189026", + "object_id": "1588", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513939991", + "to_ids": false, + "type": "text", + "uuid": "5a3ce417-43f0-494d-ac2e-55ea950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Object": { + "distribution": "5", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "uuid": "5a3ce66e-70b4-47e7-b965-46f6950d210f" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "187", + "object_id": "1588", + "object_uuid": "5a3ce417-7cd4-4c36-8a73-55ea950d210f", + "referenced_id": "1599", + "referenced_type": "1", + "referenced_uuid": "5a3ce66e-70b4-47e7-b965-46f6950d210f", + "relationship_type": "communicates-with", + "timestamp": "1513948483", + "uuid": "5a3d0543-8f74-4086-aafc-418a950d210f" + } + ], + "comment": "Win32/Sednit.BN", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1588", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513948498", + "uuid": "5a3ce417-7cd4-4c36-8a73-55ea950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189027", + "object_id": "1589", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513940012", + "to_ids": true, + "type": "sha1", + "uuid": "5a3ce42c-836c-49e7-a9f3-4a5f950d210f", + "value": "f9fd3f1d8da4ffd6a494228b934549d09e3c59d1" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189028", + "object_id": "1589", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513940012", + "to_ids": false, + "type": "text", + "uuid": "5a3ce42c-4c88-4940-94b8-4084950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Object": { + "distribution": "5", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "uuid": "5a3ce60a-6db8-4212-b194-4339950d210f" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "183", + "object_id": "1589", + "object_uuid": "5a3ce42b-2e0c-4a26-b6c8-47a3950d210f", + "referenced_id": "1594", + "referenced_type": "1", + "referenced_uuid": "5a3ce60a-6db8-4212-b194-4339950d210f", + "relationship_type": "communicates-with", + "timestamp": "1513948106", + "uuid": "5a3d03ca-2398-4060-b13c-404a950d210f" + }, + { + "Object": { + "distribution": "5", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "uuid": "5a3ce61a-c1f0-4c7c-b815-4fa9950d210f" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "184", + "object_id": "1589", + "object_uuid": "5a3ce42b-2e0c-4a26-b6c8-47a3950d210f", + "referenced_id": "1595", + "referenced_type": "1", + "referenced_uuid": "5a3ce61a-c1f0-4c7c-b815-4fa9950d210f", + "relationship_type": "communicates-with", + "timestamp": "1513948117", + "uuid": "5a3d03d5-6d8c-4dfb-b193-4002950d210f" + } + ], + "comment": "Win32/Sednit.BN", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1589", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513948128", + "uuid": "5a3ce42b-2e0c-4a26-b6c8-47a3950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189029", + "object_id": "1590", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513940027", + "to_ids": true, + "type": "sha1", + "uuid": "5a3ce43b-6738-4a14-a318-4d65950d210f", + "value": "e338d49c270baf64363879e5eecb8fa6bdde8ad9" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189030", + "object_id": "1590", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513940027", + "to_ids": false, + "type": "text", + "uuid": "5a3ce43b-3a10-4d78-9ee2-485c950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Object": { + "distribution": "5", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "uuid": "5a3ce5f8-3418-4f7b-ae41-4bca950d210f" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "186", + "object_id": "1590", + "object_uuid": "5a3ce43a-5478-4f65-95b2-4e1e950d210f", + "referenced_id": "1593", + "referenced_type": "1", + "referenced_uuid": "5a3ce5f8-3418-4f7b-ae41-4bca950d210f", + "relationship_type": "communicates-with", + "timestamp": "1513948320", + "uuid": "5a3d04a0-9d28-47c3-a12c-465b950d210f" + } + ], + "comment": "Win32/Sednit.BG", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1590", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513948339", + "uuid": "5a3ce43a-5478-4f65-95b2-4e1e950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189031", + "object_id": "1591", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513940042", + "to_ids": true, + "type": "sha1", + "uuid": "5a3ce44a-2ea4-4526-8bbc-c328950d210f", + "value": "6e167da3c5d887fa2e58da848a2245d11b6c5ad6" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189032", + "object_id": "1591", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513940042", + "to_ids": false, + "type": "text", + "uuid": "5a3ce44a-5118-4142-97f0-c328950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Object": { + "distribution": "5", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "uuid": "5a3ce64e-8bf8-4dc6-be49-437f950d210f" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "170", + "object_id": "1591", + "object_uuid": "5a3ce44a-ce70-42b7-80b8-c328950d210f", + "referenced_id": "1597", + "referenced_type": "1", + "referenced_uuid": "5a3ce64e-8bf8-4dc6-be49-437f950d210f", + "relationship_type": "communicates-with", + "timestamp": "1513940734", + "uuid": "5a3ce6fe-b0c4-44df-a609-419a950d210f" + }, + { + "Object": { + "distribution": "5", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "uuid": "5a3ce65c-fc40-4585-817e-4ca3950d210f" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "171", + "object_id": "1591", + "object_uuid": "5a3ce44a-ce70-42b7-80b8-c328950d210f", + "referenced_id": "1598", + "referenced_type": "1", + "referenced_uuid": "5a3ce65c-fc40-4585-817e-4ca3950d210f", + "relationship_type": "communicates-with", + "timestamp": "1513940753", + "uuid": "5a3ce711-a0dc-4dbe-b59e-495a950d210f" + } + ], + "comment": "Win32/Sednit.BG", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1591", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513940753", + "uuid": "5a3ce44a-ce70-42b7-80b8-c328950d210f" + }, + { + "Attribute": [ + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189033", + "object_id": "1592", + "object_relation": "ip", + "sharing_group_id": "0", + "timestamp": "1513940362", + "to_ids": true, + "type": "ip-dst", + "uuid": "5a3ce58a-fcd8-48d5-8b4a-4fd9950d210f", + "value": "87.236.211.182" + }, + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189034", + "object_id": "1592", + "object_relation": "domain", + "sharing_group_id": "0", + "timestamp": "1513940362", + "to_ids": true, + "type": "domain", + "uuid": "5a3ce58a-6e14-48ea-9746-48f2950d210f", + "value": "servicecdp.com" + } + ], + "comment": "", + "deleted": false, + "description": "A domain and IP address seen as a tuple in a specific time frame.", + "distribution": "5", + "event_id": "9747", + "id": "1592", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "template_uuid": "43b3b146-77eb-4931-b4cc-b66c60f28734", + "template_version": "5", + "timestamp": "1513940362", + "uuid": "5a3ce58a-3198-4cb8-9d51-44e5950d210f" + }, + { + "Attribute": [ + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189035", + "object_id": "1593", + "object_relation": "ip", + "sharing_group_id": "0", + "timestamp": "1513940472", + "to_ids": true, + "type": "ip-dst", + "uuid": "5a3ce5f8-99b4-41a2-915a-4bf8950d210f", + "value": "95.215.45.43" + }, + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189036", + "object_id": "1593", + "object_relation": "domain", + "sharing_group_id": "0", + "timestamp": "1513940472", + "to_ids": true, + "type": "domain", + "uuid": "5a3ce5f8-62c8-4f04-89c2-4aeb950d210f", + "value": "wmdmediacodecs.com" + } + ], + "comment": "", + "deleted": false, + "description": "A domain and IP address seen as a tuple in a specific time frame.", + "distribution": "5", + "event_id": "9747", + "id": "1593", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "template_uuid": "43b3b146-77eb-4931-b4cc-b66c60f28734", + "template_version": "5", + "timestamp": "1513940472", + "uuid": "5a3ce5f8-3418-4f7b-ae41-4bca950d210f" + }, + { + "Attribute": [ + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189037", + "object_id": "1594", + "object_relation": "ip", + "sharing_group_id": "0", + "timestamp": "1513940490", + "to_ids": true, + "type": "ip-dst", + "uuid": "5a3ce60a-cc50-4553-bfff-4ea9950d210f", + "value": "89.45.67.144" + }, + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189038", + "object_id": "1594", + "object_relation": "domain", + "sharing_group_id": "0", + "timestamp": "1513940491", + "to_ids": true, + "type": "domain", + "uuid": "5a3ce60b-e648-4667-8432-4ba8950d210f", + "value": "mvband.net" + } + ], + "comment": "", + "deleted": false, + "description": "A domain and IP address seen as a tuple in a specific time frame.", + "distribution": "5", + "event_id": "9747", + "id": "1594", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "template_uuid": "43b3b146-77eb-4931-b4cc-b66c60f28734", + "template_version": "5", + "timestamp": "1513940490", + "uuid": "5a3ce60a-6db8-4212-b194-4339950d210f" + }, + { + "Attribute": [ + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189039", + "object_id": "1595", + "object_relation": "ip", + "sharing_group_id": "0", + "timestamp": "1513940506", + "to_ids": true, + "type": "ip-dst", + "uuid": "5a3ce61a-4458-4c36-866e-44e9950d210f", + "value": "89.33.246.117" + }, + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189040", + "object_id": "1595", + "object_relation": "domain", + "sharing_group_id": "0", + "timestamp": "1513940506", + "to_ids": true, + "type": "domain", + "uuid": "5a3ce61a-f820-4a43-b3d9-47e5950d210f", + "value": "mvtband.net" + } + ], + "comment": "", + "deleted": false, + "description": "A domain and IP address seen as a tuple in a specific time frame.", + "distribution": "5", + "event_id": "9747", + "id": "1595", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "template_uuid": "43b3b146-77eb-4931-b4cc-b66c60f28734", + "template_version": "5", + "timestamp": "1513940506", + "uuid": "5a3ce61a-c1f0-4c7c-b815-4fa9950d210f" + }, + { + "Attribute": [ + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189041", + "object_id": "1596", + "object_relation": "ip", + "sharing_group_id": "0", + "timestamp": "1513940542", + "to_ids": true, + "type": "ip-dst", + "uuid": "5a3ce63e-66d4-483f-bae6-44f6950d210f", + "value": "87.236.211.182" + }, + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189042", + "object_id": "1596", + "object_relation": "domain", + "sharing_group_id": "0", + "timestamp": "1513940542", + "to_ids": true, + "type": "domain", + "uuid": "5a3ce63e-0d88-405b-82a9-43b5950d210f", + "value": "servicecdp.com" + } + ], + "comment": "", + "deleted": false, + "description": "A domain and IP address seen as a tuple in a specific time frame.", + "distribution": "5", + "event_id": "9747", + "id": "1596", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "template_uuid": "43b3b146-77eb-4931-b4cc-b66c60f28734", + "template_version": "5", + "timestamp": "1513940542", + "uuid": "5a3ce63e-0240-46f5-b9ed-4759950d210f" + }, + { + "Attribute": [ + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189043", + "object_id": "1597", + "object_relation": "ip", + "sharing_group_id": "0", + "timestamp": "1513940558", + "to_ids": true, + "type": "ip-dst", + "uuid": "5a3ce64e-d7a8-4817-a132-4c72950d210f", + "value": "185.156.173.70" + }, + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189044", + "object_id": "1597", + "object_relation": "domain", + "sharing_group_id": "0", + "timestamp": "1513940558", + "to_ids": true, + "type": "domain", + "uuid": "5a3ce64e-243c-4931-b733-403c950d210f", + "value": "runvercheck.com" + } + ], + "comment": "", + "deleted": false, + "description": "A domain and IP address seen as a tuple in a specific time frame.", + "distribution": "5", + "event_id": "9747", + "id": "1597", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "template_uuid": "43b3b146-77eb-4931-b4cc-b66c60f28734", + "template_version": "5", + "timestamp": "1513940558", + "uuid": "5a3ce64e-8bf8-4dc6-be49-437f950d210f" + }, + { + "Attribute": [ + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189045", + "object_id": "1598", + "object_relation": "ip", + "sharing_group_id": "0", + "timestamp": "1513940572", + "to_ids": true, + "type": "ip-dst", + "uuid": "5a3ce65c-bf78-4b78-bafd-4cf6950d210f", + "value": "191.101.31.96" + }, + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189046", + "object_id": "1598", + "object_relation": "domain", + "sharing_group_id": "0", + "timestamp": "1513940572", + "to_ids": true, + "type": "domain", + "uuid": "5a3ce65c-8140-4146-a927-45e4950d210f", + "value": "remsupport.org" + } + ], + "comment": "", + "deleted": false, + "description": "A domain and IP address seen as a tuple in a specific time frame.", + "distribution": "5", + "event_id": "9747", + "id": "1598", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "template_uuid": "43b3b146-77eb-4931-b4cc-b66c60f28734", + "template_version": "5", + "timestamp": "1513940572", + "uuid": "5a3ce65c-fc40-4585-817e-4ca3950d210f" + }, + { + "Attribute": [ + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189047", + "object_id": "1599", + "object_relation": "ip", + "sharing_group_id": "0", + "timestamp": "1513940591", + "to_ids": true, + "type": "ip-dst", + "uuid": "5a3ce66f-150c-43ec-a3ff-4aa5950d210f", + "value": "89.187.150.44" + }, + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189048", + "object_id": "1599", + "object_relation": "domain", + "sharing_group_id": "0", + "timestamp": "1513940591", + "to_ids": true, + "type": "domain", + "uuid": "5a3ce66f-466c-478e-8064-4b42950d210f", + "value": "viters.org" + } + ], + "comment": "", + "deleted": false, + "description": "A domain and IP address seen as a tuple in a specific time frame.", + "distribution": "5", + "event_id": "9747", + "id": "1599", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "template_uuid": "43b3b146-77eb-4931-b4cc-b66c60f28734", + "template_version": "5", + "timestamp": "1513940590", + "uuid": "5a3ce66e-70b4-47e7-b965-46f6950d210f" + }, + { + "Attribute": [ + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189049", + "object_id": "1600", + "object_relation": "ip", + "sharing_group_id": "0", + "timestamp": "1513940608", + "to_ids": true, + "type": "ip-dst", + "uuid": "5a3ce680-7b04-466d-b187-4301950d210f", + "value": "146.185.253.132" + }, + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189050", + "object_id": "1600", + "object_relation": "domain", + "sharing_group_id": "0", + "timestamp": "1513940608", + "to_ids": true, + "type": "domain", + "uuid": "5a3ce680-12f4-4001-9f86-4aa4950d210f", + "value": "myinvestgroup.com" + } + ], + "comment": "", + "deleted": false, + "description": "A domain and IP address seen as a tuple in a specific time frame.", + "distribution": "5", + "event_id": "9747", + "id": "1600", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "template_uuid": "43b3b146-77eb-4931-b4cc-b66c60f28734", + "template_version": "5", + "timestamp": "1513940608", + "uuid": "5a3ce680-90d4-478d-95db-48a6950d210f" + }, + { + "Attribute": [ + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189051", + "object_id": "1601", + "object_relation": "ip", + "sharing_group_id": "0", + "timestamp": "1513940621", + "to_ids": true, + "type": "ip-dst", + "uuid": "5a3ce68d-0108-4557-8921-4377950d210f", + "value": "86.106.131.141" + }, + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189052", + "object_id": "1601", + "object_relation": "domain", + "sharing_group_id": "0", + "timestamp": "1513940622", + "to_ids": true, + "type": "domain", + "uuid": "5a3ce68e-54d0-4c67-8c4c-4dea950d210f", + "value": "space-delivery.com" + } + ], + "comment": "", + "deleted": false, + "description": "A domain and IP address seen as a tuple in a specific time frame.", + "distribution": "5", + "event_id": "9747", + "id": "1601", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "template_uuid": "43b3b146-77eb-4931-b4cc-b66c60f28734", + "template_version": "5", + "timestamp": "1513940621", + "uuid": "5a3ce68d-1940-4ea6-becd-44fe950d210f" + }, + { + "Attribute": [ + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189054", + "object_id": "1602", + "object_relation": "ip", + "sharing_group_id": "0", + "timestamp": "1513940642", + "to_ids": true, + "type": "ip-dst", + "uuid": "5a3ce6a2-4a38-4b90-8d74-4f10950d210f", + "value": "89.34.111.160" + }, + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189055", + "object_id": "1602", + "object_relation": "domain", + "sharing_group_id": "0", + "timestamp": "1513940642", + "to_ids": true, + "type": "domain", + "uuid": "5a3ce6a2-ffa4-4afb-89ab-42a6950d210f", + "value": "satellitedeluxpanorama.com" + } + ], + "comment": "", + "deleted": false, + "description": "A domain and IP address seen as a tuple in a specific time frame.", + "distribution": "5", + "event_id": "9747", + "id": "1602", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "template_uuid": "43b3b146-77eb-4931-b4cc-b66c60f28734", + "template_version": "5", + "timestamp": "1513940641", + "uuid": "5a3ce6a1-3f1c-4d5d-bac7-406d950d210f" + }, + { + "Attribute": [ + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189056", + "object_id": "1603", + "object_relation": "ip", + "sharing_group_id": "0", + "timestamp": "1513940654", + "to_ids": true, + "type": "ip-dst", + "uuid": "5a3ce6ae-601c-44b8-8eec-4a5f950d210f", + "value": "185.216.35.26" + }, + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189057", + "object_id": "1603", + "object_relation": "domain", + "sharing_group_id": "0", + "timestamp": "1513940654", + "to_ids": true, + "type": "domain", + "uuid": "5a3ce6ae-3b00-420a-82fd-45fb950d210f", + "value": "webviewres.net" + } + ], + "comment": "", + "deleted": false, + "description": "A domain and IP address seen as a tuple in a specific time frame.", + "distribution": "5", + "event_id": "9747", + "id": "1603", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "template_uuid": "43b3b146-77eb-4931-b4cc-b66c60f28734", + "template_version": "5", + "timestamp": "1513940654", + "uuid": "5a3ce6ae-98d8-4270-b88f-47f2950d210f" + } + ], + "Org": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "Orgc": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "RelatedEvent": [ + { + "Event": { + "Org": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "Orgc": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "analysis": "2", + "date": "2017-12-14", + "distribution": "3", + "id": "9616", + "info": "OSINT - Attackers Deploy New ICS Attack Framework \u201cTRITON\u201d and Cause Operational Disruption to Critical Infrastructure", + "org_id": "2", + "orgc_id": "2", + "published": false, + "threat_level_id": "3", + "timestamp": "1513674510", + "uuid": "5a329d19-03e0-4eaa-8b4d-4310950d210f" + } + }, + { + "Event": { + "Org": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "Orgc": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "analysis": "2", + "date": "2017-12-07", + "distribution": "3", + "id": "9552", + "info": "OSINT - Master Channel: The Boleto Mestre Campaign Targets Brazil", + "org_id": "2", + "orgc_id": "2", + "published": false, + "threat_level_id": "3", + "timestamp": "1512657975", + "uuid": "5a2943a3-c574-44bb-8e68-45de950d210f" + } + }, + { + "Event": { + "Org": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "Orgc": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "analysis": "0", + "date": "2017-11-27", + "distribution": "3", + "id": "9513", + "info": "OSINT - Tizi: Detecting and blocking socially engineered spyware on Android", + "org_id": "2", + "orgc_id": "2", + "published": true, + "threat_level_id": "3", + "timestamp": "1512356440", + "uuid": "5a23a972-e6a0-4a05-b505-4e8f02de0b81" + } + }, + { + "Event": { + "Org": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "Orgc": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "analysis": "2", + "date": "2017-11-07", + "distribution": "3", + "id": "9309", + "info": "OSINT - Threat Group APT28 Slips Office Malware into Doc Citing NYC Terror Attack", + "org_id": "2", + "orgc_id": "2", + "published": true, + "threat_level_id": "3", + "timestamp": "1511385862", + "uuid": "5a021bc2-8e0c-4ac5-b048-cc3e02de0b81" + } + }, + { + "Event": { + "Org": { + "id": "291", + "name": "NCSC-NL", + "uuid": "5697b0c4-9474-4336-b675-28140a950b0b" + }, + "Orgc": { + "id": "291", + "name": "NCSC-NL", + "uuid": "5697b0c4-9474-4336-b675-28140a950b0b" + }, + "analysis": "2", + "date": "2017-10-23", + "distribution": "3", + "id": "9208", + "info": "Talos: \u201cCyber Conflict\u201d Decoy Document Used In Real Cyber Conflict", + "org_id": "291", + "orgc_id": "291", + "published": true, + "threat_level_id": "2", + "timestamp": "1510088616", + "uuid": "59ed9c81-6484-47a9-aab4-191d0a950b0c" + } + }, + { + "Event": { + "Org": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "Orgc": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "analysis": "2", + "date": "2017-08-11", + "distribution": "3", + "id": "8798", + "info": "OSINT - APT28 Targets Hospitality Sector, Presents Threat to Travelers", + "org_id": "2", + "orgc_id": "2", + "published": true, + "threat_level_id": "3", + "timestamp": "1502460096", + "uuid": "598db7fd-47a8-45f8-9414-408b02de0b81" + } + }, + { + "Event": { + "Org": { + "id": "231", + "name": "kingfisherops.com", + "uuid": "566ff5f4-7020-4089-9003-4374950d210f" + }, + "Orgc": { + "id": "204", + "name": "CERT-BUND", + "uuid": "56a64d7a-63dc-4471-bce9-4accc25ed029" + }, + "analysis": "0", + "date": "2017-07-25", + "distribution": "3", + "id": "8750", + "info": "European Defence Agency lure drops mssuppa.dat", + "org_id": "231", + "orgc_id": "204", + "published": true, + "threat_level_id": "2", + "timestamp": "1500967989", + "uuid": "5976f294-a844-44fe-a4f0-6c67c25ed029" + } + }, + { + "Event": { + "Org": { + "id": "277", + "name": "inthreat.com", + "uuid": "5697b91d-2090-441f-b153-75e895ca48b7" + }, + "Orgc": { + "id": "277", + "name": "inthreat.com", + "uuid": "5697b91d-2090-441f-b153-75e895ca48b7" + }, + "analysis": "2", + "date": "2017-05-11", + "distribution": "3", + "id": "7820", + "info": "APT28-Sednit adds two zero-day exploits using \u2018Trump\u2019s attack on Syria\u2019 as a decoy", + "org_id": "277", + "orgc_id": "277", + "published": true, + "threat_level_id": "2", + "timestamp": "1494824291", + "uuid": "59147a22-3100-4779-9377-360395ca48b7" + } + }, + { + "Event": { + "Org": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "Orgc": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "analysis": "2", + "date": "2017-05-09", + "distribution": "3", + "id": "7801", + "info": "OSINT - EPS Processing Zero-Days Exploited by Multiple Threat Actors", + "org_id": "2", + "orgc_id": "2", + "published": true, + "threat_level_id": "3", + "timestamp": "1494354378", + "uuid": "59120865-27e0-4e6d-9b74-4a9f950d210f" + } + }, + { + "Event": { + "Org": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "Orgc": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "analysis": "0", + "date": "2016-12-29", + "distribution": "3", + "id": "5667", + "info": "OSINT - GRIZZLY STEPPE \u2013 Russian Malicious Cyber Activity", + "org_id": "2", + "orgc_id": "2", + "published": true, + "threat_level_id": "3", + "timestamp": "1494853878", + "uuid": "58658c15-54ac-43c3-9beb-414502de0b81" + } + }, + { + "Event": { + "Org": { + "id": "277", + "name": "inthreat.com", + "uuid": "5697b91d-2090-441f-b153-75e895ca48b7" + }, + "Orgc": { + "id": "277", + "name": "inthreat.com", + "uuid": "5697b91d-2090-441f-b153-75e895ca48b7" + }, + "analysis": "2", + "date": "2016-12-20", + "distribution": "1", + "id": "5616", + "info": "APT28-The Sofacy Group's DealersChoice Attacks Continue", + "org_id": "277", + "orgc_id": "277", + "published": true, + "threat_level_id": "2", + "timestamp": "1494829249", + "uuid": "58594faf-e98c-4c03-a58c-43cf95ca48b7" + } + }, + { + "Event": { + "Org": { + "id": "291", + "name": "NCSC-NL", + "uuid": "5697b0c4-9474-4336-b675-28140a950b0b" + }, + "Orgc": { + "id": "291", + "name": "NCSC-NL", + "uuid": "5697b0c4-9474-4336-b675-28140a950b0b" + }, + "analysis": "1", + "date": "2016-11-09", + "distribution": "3", + "id": "5348", + "info": "[APT-28/Sofacy]Pawn Storm Ramps Up [European Government] Spear-phishing Before Zero-Days Get Patched", + "org_id": "291", + "orgc_id": "291", + "published": true, + "threat_level_id": "1", + "timestamp": "1481709638", + "uuid": "582341ff-0830-4b32-aaba-08640a950b0c" + } + }, + { + "Event": { + "Org": { + "id": "74", + "name": "PwC.lu", + "uuid": "55f6ea61-4f74-40b6-a6df-4ff9950d210f" + }, + "Orgc": { + "id": "325", + "name": "CUDESO", + "uuid": "56c42374-fdb8-4544-a218-41ffc0a8ab16" + }, + "analysis": "2", + "date": "2016-11-09", + "distribution": "3", + "id": "5641", + "info": "Pawn Storm Ramps Up Spear-phishing Before Zero-Days Get Patched", + "org_id": "74", + "orgc_id": "325", + "published": true, + "threat_level_id": "2", + "timestamp": "1478712711", + "uuid": "58235d0e-34d4-41c1-9a2e-04dcc0a8ab16" + } + }, + { + "Event": { + "Org": { + "id": "335", + "name": "Orange CERT-CC", + "uuid": "5707ccb5-e330-4e25-a193-41d4950d210f" + }, + "Orgc": { + "id": "335", + "name": "Orange CERT-CC", + "uuid": "5707ccb5-e330-4e25-a193-41d4950d210f" + }, + "analysis": "0", + "date": "2016-10-18", + "distribution": "0", + "id": "5163", + "info": "Orange-CERT-CC Test #01", + "org_id": "335", + "orgc_id": "335", + "published": false, + "threat_level_id": "3", + "timestamp": "1476782422", + "uuid": "5805e8a5-611c-498b-839b-bd57950d210f" + } + }, + { + "Event": { + "Org": { + "id": "278", + "name": "TDC.dk", + "uuid": "56a5d575-2ff4-4738-a2ee-59be950d210f" + }, + "Orgc": { + "id": "278", + "name": "TDC.dk", + "uuid": "56a5d575-2ff4-4738-a2ee-59be950d210f" + }, + "analysis": "2", + "date": "2016-10-17", + "distribution": "3", + "id": "5165", + "info": "OSINT: \u2018DealersChoice\u2019 is Sofacy\u2019s Flash Player Exploit Platform", + "org_id": "278", + "orgc_id": "278", + "published": true, + "threat_level_id": "1", + "timestamp": "1476789563", + "uuid": "580602f6-f8b8-4ac3-9813-7bf7bce2ab96" + } + }, + { + "Event": { + "Org": { + "id": "412", + "name": "TS", + "uuid": "57470e61-3384-491d-a56f-1bb75b86d7e5" + }, + "Orgc": { + "id": "412", + "name": "TS", + "uuid": "57470e61-3384-491d-a56f-1bb75b86d7e5" + }, + "analysis": "2", + "date": "2016-08-19", + "distribution": "1", + "id": "4710", + "info": "bullettin.doc sample, linked to APT28 campaign", + "org_id": "412", + "orgc_id": "412", + "published": true, + "threat_level_id": "1", + "timestamp": "1476776982", + "uuid": "57b7248f-283c-442e-8e02-2d0f5b86d7e5" + } + }, + { + "Event": { + "Org": { + "id": "277", + "name": "inthreat.com", + "uuid": "5697b91d-2090-441f-b153-75e895ca48b7" + }, + "Orgc": { + "id": "277", + "name": "inthreat.com", + "uuid": "5697b91d-2090-441f-b153-75e895ca48b7" + }, + "analysis": "2", + "date": "2016-06-20", + "distribution": "3", + "id": "4172", + "info": "APT28 and APT29 - Inside the DNC Breaches", + "org_id": "277", + "orgc_id": "277", + "published": true, + "threat_level_id": "2", + "timestamp": "1494829231", + "uuid": "5767c102-c170-4124-ae3d-7bef95ca48b7" + } + }, + { + "Event": { + "Org": { + "id": "347", + "name": "incibe.es", + "uuid": "5720623c-129c-4989-ae9d-4a11950d210f" + }, + "Orgc": { + "id": "665", + "name": "INCIBE", + "uuid": "56fa4fe4-f528-4480-8332-1ba3c0a80a8c" + }, + "analysis": "2", + "date": "2016-06-16", + "distribution": "3", + "id": "6131", + "info": "New Sofacy (APT28) attacks against a US Government Agency", + "org_id": "347", + "orgc_id": "665", + "published": true, + "threat_level_id": "1", + "timestamp": "1488792538", + "uuid": "5762a86a-e314-4e4e-ba5a-51c5c0a80a8e" + } + }, + { + "Event": { + "Org": { + "id": "26", + "name": "CthulhuSPRL.be", + "uuid": "55f6ea5f-fd34-43b8-ac1d-40cb950d210f" + }, + "Orgc": { + "id": "26", + "name": "CthulhuSPRL.be", + "uuid": "55f6ea5f-fd34-43b8-ac1d-40cb950d210f" + }, + "analysis": "2", + "date": "2016-06-15", + "distribution": "3", + "id": "3987", + "info": "OSINT New Sofacy Attacks Against US Government Agency by Palo Alto Unit 42", + "org_id": "26", + "orgc_id": "26", + "published": true, + "threat_level_id": "1", + "timestamp": "1466000907", + "uuid": "57613790-f6b4-4895-943f-4467950d210f" + } + }, + { + "Event": { + "Org": { + "id": "278", + "name": "TDC.dk", + "uuid": "56a5d575-2ff4-4738-a2ee-59be950d210f" + }, + "Orgc": { + "id": "325", + "name": "CUDESO", + "uuid": "56c42374-fdb8-4544-a218-41ffc0a8ab16" + }, + "analysis": "2", + "date": "2016-06-14", + "distribution": "3", + "id": "4183", + "info": "New Sofacy Attacks Against US Government Agency", + "org_id": "278", + "orgc_id": "325", + "published": true, + "threat_level_id": "2", + "timestamp": "1467289109", + "uuid": "57607369-2490-444a-9034-049fc0a8ab16" + } + } + ], + "Tag": [ + { + "colour": "#00d622", + "exportable": true, + "hide_tag": false, + "id": "2", + "name": "tlp:white", + "user_id": "0" + }, + { + "colour": "#ef0081", + "exportable": true, + "hide_tag": false, + "id": "2986", + "name": "workflow:state=\"incomplete\"", + "user_id": "0" + }, + { + "colour": "#810046", + "exportable": true, + "hide_tag": false, + "id": "2979", + "name": "workflow:todo=\"create-missing-misp-galaxy-cluster-values\"", + "user_id": "0" + }, + { + "colour": "#91004e", + "exportable": true, + "hide_tag": false, + "id": "2980", + "name": "workflow:todo=\"create-missing-misp-galaxy-cluster\"", + "user_id": "0" + }, + { + "colour": "#12e000", + "exportable": true, + "hide_tag": false, + "id": "1100", + "name": "misp-galaxy:threat-actor=\"Sofacy\"", + "user_id": "0" + }, + { + "colour": "#0088cc", + "exportable": true, + "hide_tag": false, + "id": "3007", + "name": "misp-galaxy:exploit-kit=\"Sednit EK\"", + "user_id": "0" + }, + { + "colour": "#0088cc", + "exportable": true, + "hide_tag": false, + "id": "2215", + "name": "misp-galaxy:tool=\"GAMEFISH\"", + "user_id": "0" + }, + { + "colour": "#0088cc", + "exportable": true, + "hide_tag": false, + "id": "3008", + "name": "misp-galaxy:mitre-malware=\"JHUHUGIT\"", + "user_id": "0" + }, + { + "colour": "#0c9900", + "exportable": true, + "hide_tag": false, + "id": "1012", + "name": "misp-galaxy:tool=\"X-Tunnel\"", + "user_id": "0" + }, + { + "colour": "#0088cc", + "exportable": true, + "hide_tag": false, + "id": "3009", + "name": "misp-galaxy:mitre-malware=\"XTunnel\"", + "user_id": "0" + }, + { + "colour": "#0088cc", + "exportable": true, + "hide_tag": false, + "id": "3010", + "name": "misp-galaxy:mitre-malware=\"ADVSTORESHELL\"", + "user_id": "0" + }, + { + "colour": "#0088cc", + "exportable": true, + "hide_tag": false, + "id": "3011", + "name": "misp-galaxy:tool=\"EVILTOSS\"", + "user_id": "0" + }, + { + "colour": "#0088cc", + "exportable": true, + "hide_tag": false, + "id": "3012", + "name": "misp-galaxy:mitre-malware=\"USBStealer\"", + "user_id": "0" + }, + { + "colour": "#0c9800", + "exportable": true, + "hide_tag": false, + "id": "1011", + "name": "misp-galaxy:tool=\"X-Agent\"", + "user_id": "0" + }, + { + "colour": "#0088cc", + "exportable": true, + "hide_tag": false, + "id": "3013", + "name": "misp-galaxy:mitre-malware=\"XAgentOSX\"", + "user_id": "0" + }, + { + "colour": "#0088cc", + "exportable": true, + "hide_tag": false, + "id": "3014", + "name": "misp-galaxy:mitre-malware=\"CHOPSTICK\"", + "user_id": "0" + }, + { + "colour": "#0088cc", + "exportable": true, + "hide_tag": false, + "id": "3015", + "name": "misp-galaxy:exploit-kit=\"DealersChoice\"", + "user_id": "0" + }, + { + "colour": "#0088cc", + "exportable": true, + "hide_tag": false, + "id": "3016", + "name": "misp-galaxy:mitre-malware=\"Downdelph\"", + "user_id": "0" + } + ], + "analysis": "0", + "attribute_count": "122", + "date": "2017-12-21", + "disable_correlation": false, + "distribution": "3", + "event_creator_email": "alexandre.dulaunoy@circl.lu", + "id": "9747", + "info": "OSINT - Sednit update: How Fancy Bear Spent the Year", + "locked": false, + "org_id": "2", + "orgc_id": "2", + "proposal_email_lock": false, + "publish_timestamp": "0", + "published": false, + "sharing_group_id": "0", + "threat_level_id": "3", + "timestamp": "1513948642", + "uuid": "5a3c2fcd-8328-42bb-a95e-4f4402de0b81" + } +} diff --git a/tests/mispevent_testfiles/malware.json b/tests/mispevent_testfiles/malware.json new file mode 100644 index 0000000..3f7545d --- /dev/null +++ b/tests/mispevent_testfiles/malware.json @@ -0,0 +1,21 @@ +{ + "Event": { + "Attribute": [ + { + "category": "Payload delivery", + "data": "ewogICJFdmVudCI6IHsKICB9Cn0K", + "disable_correlation": false, + "encrypt": true, + "malware_filename": "bar.exe", + "to_ids": true, + "type": "malware-sample", + "value": "bar.exe|7637beddacbeac59d44469b2b120b9e6" + } + ], + "analysis": "1", + "date": "2017-12-31", + "distribution": "1", + "info": "This is a test", + "threat_level_id": "1" + } +} diff --git a/tests/mispevent_testfiles/proposals.json b/tests/mispevent_testfiles/proposals.json new file mode 100644 index 0000000..e249fd6 --- /dev/null +++ b/tests/mispevent_testfiles/proposals.json @@ -0,0 +1,36 @@ +{ + "Event": { + "Attribute": [ + { + "ShadowAttribute": [ + { + "category": "Payload delivery", + "disable_correlation": false, + "to_ids": true, + "type": "filename", + "value": "bar.pdf" + } + ], + "category": "Payload delivery", + "disable_correlation": false, + "to_ids": true, + "type": "filename", + "value": "bar.exe" + } + ], + "ShadowAttribute": [ + { + "category": "Payload delivery", + "disable_correlation": false, + "to_ids": true, + "type": "filename", + "value": "baz.jpg" + } + ], + "analysis": "1", + "date": "2017-12-31", + "distribution": "1", + "info": "This is a test", + "threat_level_id": "1" + } +} diff --git a/tests/mispevent_testfiles/shadow.json b/tests/mispevent_testfiles/shadow.json new file mode 100644 index 0000000..bc0f053 --- /dev/null +++ b/tests/mispevent_testfiles/shadow.json @@ -0,0 +1,149 @@ +{ + "Event": { + "Attribute": [ + { + "ShadowAttribute": [ + { + "Org": { + "id": "1", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "category": "Artifacts dropped", + "comment": "", + "disable_correlation": false, + "event_id": "6676", + "event_uuid": "5a4cb19a-f550-437f-bd29-48ed950d210f", + "id": "3770", + "old_id": "811578", + "org_id": "1", + "proposal_to_delete": false, + "timestamp": "1514975846", + "to_ids": true, + "type": "filename", + "uuid": "5a4cb1c7-fa84-45fa-8d27-4822950d210f", + "value": "blah.exe.jpg" + } + ], + "category": "Artifacts dropped", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "6676", + "id": "811578", + "object_id": "0", + "sharing_group_id": "0", + "timestamp": "1514975687", + "to_ids": false, + "type": "filename", + "uuid": "5a4cb1c7-fa84-45fa-8d27-4822950d210f", + "value": "blah.exe" + } + ], + "Object": [ + { + "Attribute": [ + { + "ShadowAttribute": [ + { + "Org": { + "id": "1", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "category": "Payload delivery", + "comment": "", + "disable_correlation": false, + "event_id": "6676", + "event_uuid": "5a4cb19a-f550-437f-bd29-48ed950d210f", + "id": "3771", + "old_id": "811579", + "org_id": "1", + "proposal_to_delete": false, + "timestamp": "1514976196", + "to_ids": true, + "type": "filename", + "uuid": "5a4cb2b8-4748-4c72-96e6-4588950d210f", + "value": "baz.png.exe" + } + ], + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "6676", + "id": "811579", + "object_id": "2278", + "object_relation": "filename", + "sharing_group_id": "0", + "timestamp": "1514975928", + "to_ids": true, + "type": "filename", + "uuid": "5a4cb2b8-4748-4c72-96e6-4588950d210f", + "value": "baz.png" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "6676", + "id": "811580", + "object_id": "2278", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1514975928", + "to_ids": false, + "type": "text", + "uuid": "5a4cb2b9-92b4-4d3a-82df-4e86950d210f", + "value": "Malicious" + } + ], + "comment": "", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "6676", + "id": "2278", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "7", + "timestamp": "1514975928", + "uuid": "5a4cb2b8-7958-4323-852c-4d2a950d210f" + } + ], + "Org": { + "id": "1", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "Orgc": { + "id": "1", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "analysis": "2", + "attribute_count": "3", + "date": "2018-01-03", + "disable_correlation": false, + "distribution": "0", + "event_creator_email": "raphael.vinot@circl.lu", + "id": "6676", + "info": "Test proposals / ShadowAttributes", + "locked": false, + "org_id": "1", + "orgc_id": "1", + "proposal_email_lock": true, + "publish_timestamp": "0", + "published": false, + "sharing_group_id": "0", + "threat_level_id": "1", + "timestamp": "1514975929", + "uuid": "5a4cb19a-f550-437f-bd29-48ed950d210f" + } +} diff --git a/tests/mispevent_testfiles/sighting.json b/tests/mispevent_testfiles/sighting.json new file mode 100644 index 0000000..1d7c043 --- /dev/null +++ b/tests/mispevent_testfiles/sighting.json @@ -0,0 +1,5 @@ +{ + "timestamp": 11111111, + "type": "bar", + "value": "1" +} diff --git a/tests/mispevent_testfiles/simple.json b/tests/mispevent_testfiles/simple.json new file mode 100644 index 0000000..63fbfdd --- /dev/null +++ b/tests/mispevent_testfiles/simple.json @@ -0,0 +1,4 @@ +{ + "Event": { + } +} diff --git a/tests/test_mispevent.py b/tests/test_mispevent.py new file mode 100644 index 0000000..1f6ea4a --- /dev/null +++ b/tests/test_mispevent.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import unittest +import json +from io import BytesIO + +from pymisp import MISPEvent, MISPSighting + + +class TestMISPEvent(unittest.TestCase): + + def setUp(self): + self.maxDiff = None + self.mispevent = MISPEvent() + + def init_event(self): + self.mispevent.info = 'This is a test' + self.mispevent.distribution = 1 + self.mispevent.threat_level_id = 1 + self.mispevent.analysis = 1 + self.mispevent.set_date("2017-12-31") # test the set date method + + def test_simple(self): + with open('tests/mispevent_testfiles/simple.json', 'r') as f: + ref_json = json.load(f) + self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + + def test_event(self): + self.init_event() + self.mispevent.publish() + with open('tests/mispevent_testfiles/event.json', 'r') as f: + ref_json = json.load(f) + self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + + def test_loadfile(self): + self.mispevent.load_file('tests/mispevent_testfiles/event.json') + with open('tests/mispevent_testfiles/event.json', 'r') as f: + ref_json = json.load(f) + self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + + def test_attribute(self): + self.init_event() + self.mispevent.add_attribute('filename', 'bar.exe') + self.mispevent.add_attribute_tag('osint', 'bar.exe') + attr_tags = self.mispevent.get_attribute_tag('bar.exe') + self.assertEqual(self.mispevent.attributes[0].tags[0].name, 'osint') + self.assertEqual(attr_tags[0].name, 'osint') + with open('tests/mispevent_testfiles/attribute.json', 'r') as f: + ref_json = json.load(f) + self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + # Fake setting an attribute ID for testing + self.mispevent.attributes[0].id = 42 + self.mispevent.delete_attribute(42) + with open('tests/mispevent_testfiles/attribute_del.json', 'r') as f: + ref_json = json.load(f) + self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + + def test_object_tag(self): + self.mispevent.add_object(name='file', strict=True) + self.mispevent.objects[0].add_attribute('filename', value='bar', Tag=[{'name': 'blah'}]) + self.assertEqual(self.mispevent.objects[0].attributes[0].tags[0].name, 'blah') + self.assertTrue(self.mispevent.objects[0].has_attributes_by_relation(['filename'])) + self.assertEqual(len(self.mispevent.objects[0].get_attributes_by_relation('filename')), 1) + self.mispevent.add_object(name='url', strict=True) + self.mispevent.objects[1].add_attribute('url', value='https://www.circl.lu') + self.mispevent.objects[0].uuid = 'a' + self.mispevent.objects[1].uuid = 'b' + self.mispevent.objects[0].add_reference('b', 'baz', comment='foo') + self.assertEqual(self.mispevent.objects[0].references[0].relationship_type, 'baz') + with open('tests/mispevent_testfiles/event_obj_attr_tag.json', 'r') as f: + ref_json = json.load(f) + self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + + @unittest.skip("Not supported on MISP: https://github.com/MISP/MISP/issues/2638 - https://github.com/MISP/PyMISP/issues/168") + def test_object_level_tag(self): + self.mispevent.add_object(name='file', strict=True) + self.mispevent.objects[0].add_attribute('filename', value='bar') + self.mispevent.objects[0].add_tag('osint') + self.mispevent.objects[0].uuid = 'a' + with open('tests/mispevent_testfiles/event_obj_tag.json', 'r') as f: + ref_json = json.load(f) + self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + + def test_malware(self): + with open('tests/mispevent_testfiles/simple.json', 'rb') as f: + pseudofile = BytesIO(f.read()) + self.init_event() + self.mispevent.add_attribute('malware-sample', 'bar.exe', data=pseudofile) + attribute = self.mispevent.attributes[0] + self.assertEqual(attribute.malware_binary, pseudofile) + with open('tests/mispevent_testfiles/malware.json', 'r') as f: + ref_json = json.load(f) + self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + + def test_sighting(self): + sighting = MISPSighting() + sighting.from_dict(value='1', type='bar', timestamp=11111111) + with open('tests/mispevent_testfiles/sighting.json', 'r') as f: + ref_json = json.load(f) + self.assertEqual(sighting.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + + def test_existing_event(self): + self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') + with open('tests/mispevent_testfiles/existing_event.json', 'r') as f: + ref_json = json.load(f) + self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + + def test_shadow_attributes_existing(self): + self.mispevent.load_file('tests/mispevent_testfiles/shadow.json') + with open('tests/mispevent_testfiles/shadow.json', 'r') as f: + ref_json = json.load(f) + self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + + def test_shadow_attributes(self): + self.init_event() + self.mispevent.add_proposal(type='filename', value='baz.jpg') + self.mispevent.add_attribute('filename', 'bar.exe') + self.mispevent.attributes[0].add_proposal(type='filename', value='bar.pdf') + with open('tests/mispevent_testfiles/proposals.json', 'r') as f: + ref_json = json.load(f) + self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + + def test_default_attributes(self): + self.mispevent.add_object(name='file', strict=True) + self.mispevent.objects[0].add_attribute('filename', value='bar', Tag=[{'name': 'blah'}]) + self.mispevent.add_object(name='file', strict=False, default_attributes_parameters=self.mispevent.objects[0].attributes[0]) + self.mispevent.objects[1].add_attribute('filename', value='baz') + self.mispevent.objects[0].uuid = 'a' + self.mispevent.objects[1].uuid = 'b' + with open('tests/mispevent_testfiles/event_obj_def_param.json', 'r') as f: + ref_json = json.load(f) + self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + + +if __name__ == '__main__': + unittest.main() From 53eb22cac5b64953c838ed20717fc070b5087f4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 4 Jan 2018 17:12:15 +0100 Subject: [PATCH 101/125] chg: Add test for loading existing malware sample from MISP --- tests/mispevent_testfiles/malware_exist.json | 165 +++++++++++++++++++ tests/test_mispevent.py | 8 + 2 files changed, 173 insertions(+) create mode 100644 tests/mispevent_testfiles/malware_exist.json diff --git a/tests/mispevent_testfiles/malware_exist.json b/tests/mispevent_testfiles/malware_exist.json new file mode 100644 index 0000000..9a7c3b3 --- /dev/null +++ b/tests/mispevent_testfiles/malware_exist.json @@ -0,0 +1,165 @@ +{"response":[{ + "Event": { + "id": "6719", + "orgc_id": "1", + "org_id": "1", + "date": "2018-01-04", + "threat_level_id": "1", + "info": "Test existing malware PyMISP", + "published": false, + "uuid": "5a4e4fdd-1eb4-4ff3-9e87-43fa950d210f", + "attribute_count": "6", + "analysis": "0", + "timestamp": "1515081727", + "distribution": "0", + "proposal_email_lock": false, + "locked": false, + "publish_timestamp": "0", + "sharing_group_id": "0", + "disable_correlation": false, + "event_creator_email": "raphael.vinot@circl.lu", + "Org": { + "id": "1", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "Orgc": { + "id": "1", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "Attribute": [], + "ShadowAttribute": [], + "RelatedEvent": [], + "Galaxy": [], + "Object": [ + { + "id": "2279", + "name": "file", + "meta-category": "file", + "description": "File object describing a file with meta-information", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "7", + "event_id": "6719", + "uuid": "5a4e4ffe-4cb8-48b1-bd5c-48fb950d210f", + "timestamp": "1515081726", + "distribution": "5", + "sharing_group_id": "0", + "comment": "", + "deleted": false, + "ObjectReference": [], + "Attribute": [ + { + "id": "814967", + "type": "malware-sample", + "category": "Payload delivery", + "to_ids": true, + "uuid": "5a4e4fff-407c-40ff-9de5-43dc950d210f", + "event_id": "6719", + "distribution": "5", + "timestamp": "1515081727", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "2279", + "object_relation": "malware-sample", + "value": "simple.json|7637beddacbeac59d44469b2b120b9e6", + "data": "UEsDBAoACQAAAEOAJEyjHboUIQAAABUAAAAgABwANzYzN2JlZGRhY2JlYWM1OWQ0NDQ2OWIyYjEyMGI5ZTZVVAkAA\/5PTlr+T05adXgLAAEEIQAAAAQhAAAATvzonhGOj12MyB1QeGLJ5iZhOjD+zymV4FU2+kjD4oTYUEsHCKMduhQhAAAAFQAAAFBLAwQKAAkAAABDgCRMg45UABcAAAALAAAALQAcADc2MzdiZWRkYWNiZWFjNTlkNDQ0NjliMmIxMjBiOWU2LmZpbGVuYW1lLnR4dFVUCQAD\/k9OWv5PTlp1eAsAAQQhAAAABCEAAADDgZOh6307Bduy829xtRjpivO\/xFI3KVBLBwiDjlQAFwAAAAsAAABQSwECHgMKAAkAAABDgCRMox26FCEAAAAVAAAAIAAYAAAAAAABAAAApIEAAAAANzYzN2JlZGRhY2JlYWM1OWQ0NDQ2OWIyYjEyMGI5ZTZVVAUAA\/5PTlp1eAsAAQQhAAAABCEAAABQSwECHgMKAAkAAABDgCRMg45UABcAAAALAAAALQAYAAAAAAABAAAApIGLAAAANzYzN2JlZGRhY2JlYWM1OWQ0NDQ2OWIyYjEyMGI5ZTYuZmlsZW5hbWUudHh0VVQFAAP+T05adXgLAAEEIQAAAAQhAAAAUEsFBgAAAAACAAIA2QAAABkBAAAAAA==", + "ShadowAttribute": [] + }, + { + "id": "814968", + "type": "filename", + "category": "Payload delivery", + "to_ids": false, + "uuid": "5a4e4fff-9ec0-4822-a405-4e29950d210f", + "event_id": "6719", + "distribution": "5", + "timestamp": "1515081727", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "2279", + "object_relation": "filename", + "value": "simple.json", + "ShadowAttribute": [] + }, + { + "id": "814969", + "type": "md5", + "category": "Payload delivery", + "to_ids": true, + "uuid": "5a4e4fff-8000-49f9-8c3e-4598950d210f", + "event_id": "6719", + "distribution": "5", + "timestamp": "1515081727", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "2279", + "object_relation": "md5", + "value": "7637beddacbeac59d44469b2b120b9e6", + "ShadowAttribute": [] + }, + { + "id": "814970", + "type": "sha1", + "category": "Payload delivery", + "to_ids": true, + "uuid": "5a4e4fff-dae0-4aa4-81ea-4899950d210f", + "event_id": "6719", + "distribution": "5", + "timestamp": "1515081727", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "2279", + "object_relation": "sha1", + "value": "023853a4331db8d67e44553004cf338ec1b7440e", + "ShadowAttribute": [] + }, + { + "id": "814971", + "type": "sha256", + "category": "Payload delivery", + "to_ids": true, + "uuid": "5a4e4fff-03ec-4e88-b5f4-472b950d210f", + "event_id": "6719", + "distribution": "5", + "timestamp": "1515081727", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "2279", + "object_relation": "sha256", + "value": "6ae8b0f1c7d6f3238d1fc14038018c3b4704c8cc23dac1c2bfd2c81b5a278eef", + "ShadowAttribute": [] + }, + { + "id": "814972", + "type": "size-in-bytes", + "category": "Other", + "to_ids": false, + "uuid": "5a4e4fff-b6f4-41ba-a6eb-446c950d210f", + "event_id": "6719", + "distribution": "5", + "timestamp": "1515081727", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": true, + "object_id": "2279", + "object_relation": "size-in-bytes", + "value": "21", + "ShadowAttribute": [] + } + ] + } + ] + } +}]} diff --git a/tests/test_mispevent.py b/tests/test_mispevent.py index 1f6ea4a..2cbeebf 100644 --- a/tests/test_mispevent.py +++ b/tests/test_mispevent.py @@ -93,6 +93,14 @@ class TestMISPEvent(unittest.TestCase): ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + def test_existing_malware(self): + self.mispevent.load_file('tests/mispevent_testfiles/malware_exist.json') + with open('tests/mispevent_testfiles/simple.json', 'rb') as f: + pseudofile = BytesIO(f.read()) + self.assertEqual( + self.mispevent.objects[0].get_attributes_by_relation('malware-sample')[0].malware_binary.read(), + pseudofile.read()) + def test_sighting(self): sighting = MISPSighting() sighting.from_dict(value='1', type='bar', timestamp=11111111) From 131986cf362caeb6b9833405bcdd91db9d7f2bd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 4 Jan 2018 17:28:30 +0100 Subject: [PATCH 102/125] chg: Bump misp-objects --- pymisp/data/misp-objects | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index 1460d05..875f97d 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit 1460d055a0207668cf1f0e99ff347411038f0113 +Subproject commit 875f97dce128a94d53ad13ae85c3d5e413cf1dab From b41f057855b6a8bb75b83b5087f570237c51ce15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 4 Jan 2018 17:50:52 +0100 Subject: [PATCH 103/125] chg: Fix tests (new template version) --- tests/mispevent_testfiles/event_obj_attr_tag.json | 2 +- tests/mispevent_testfiles/event_obj_def_param.json | 4 ++-- tests/mispevent_testfiles/event_obj_tag.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/mispevent_testfiles/event_obj_attr_tag.json b/tests/mispevent_testfiles/event_obj_attr_tag.json index 7d336a7..42de42e 100644 --- a/tests/mispevent_testfiles/event_obj_attr_tag.json +++ b/tests/mispevent_testfiles/event_obj_attr_tag.json @@ -31,7 +31,7 @@ "name": "file", "sharing_group_id": 0, "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", - "template_version": 8, + "template_version": 9, "uuid": "a" }, { diff --git a/tests/mispevent_testfiles/event_obj_def_param.json b/tests/mispevent_testfiles/event_obj_def_param.json index 559ba89..56bb0c5 100644 --- a/tests/mispevent_testfiles/event_obj_def_param.json +++ b/tests/mispevent_testfiles/event_obj_def_param.json @@ -23,7 +23,7 @@ "name": "file", "sharing_group_id": 0, "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", - "template_version": 8, + "template_version": 9, "uuid": "a" }, { @@ -48,7 +48,7 @@ "name": "file", "sharing_group_id": 0, "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", - "template_version": 8, + "template_version": 9, "uuid": "b" } ] diff --git a/tests/mispevent_testfiles/event_obj_tag.json b/tests/mispevent_testfiles/event_obj_tag.json index 736bff2..1542d8b 100644 --- a/tests/mispevent_testfiles/event_obj_tag.json +++ b/tests/mispevent_testfiles/event_obj_tag.json @@ -23,7 +23,7 @@ "name": "file", "sharing_group_id": 0, "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", - "template_version": 8, + "template_version": 9, "uuid": "a" } ] From 4a95a54501cd12c487a559ff9bcf9d1de7e1de80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 5 Jan 2018 11:34:08 +0100 Subject: [PATCH 104/125] fix: disable_correlation from template not properly used --- pymisp/mispevent.py | 2 +- tests/mispevent_testfiles/def_param.json | 55 ++++++++++++++++++++++++ tests/test_mispevent.py | 12 ++++++ 3 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 tests/mispevent_testfiles/def_param.json diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index cb236bf..53a36bd 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -851,7 +851,7 @@ class MISPObjectAttribute(MISPAttribute): self.type = kwargs.pop('type', None) if self.type is None: self.type = self.__definition.get('misp-attribute') - self.disable_correlation = kwargs.pop('disable_correlation', False) + self.disable_correlation = kwargs.pop('disable_correlation', None) if self.disable_correlation is None: # The correlation can be disabled by default in the object definition. # Use this value if it isn't overloaded by the object diff --git a/tests/mispevent_testfiles/def_param.json b/tests/mispevent_testfiles/def_param.json new file mode 100644 index 0000000..49a22b3 --- /dev/null +++ b/tests/mispevent_testfiles/def_param.json @@ -0,0 +1,55 @@ +{ + "Event": { + "Object": [ + { + "Attribute": [ + { + "category": "Attribution", + "disable_correlation": false, + "object_relation": "registrar", + "to_ids": false, + "type": "whois-registrar", + "value": "registar.example.com" + }, + { + "category": "Network activity", + "disable_correlation": false, + "object_relation": "domain", + "to_ids": true, + "type": "domain", + "value": "domain.example.com" + }, + { + "category": "Network activity", + "disable_correlation": true, + "object_relation": "nameserver", + "to_ids": false, + "type": "hostname", + "value": "ns1.example.com" + }, + { + "category": "External analysis", + "disable_correlation": false, + "object_relation": "nameserver", + "to_ids": true, + "type": "hostname", + "value": "ns2.example.com" + } + ], + "description": "Whois records information for a domain name.", + "distribution": 5, + "meta-category": "network", + "name": "whois", + "sharing_group_id": 0, + "template_uuid": "429faea1-34ff-47af-8a00-7c62d3be5a6a", + "template_version": 7, + "uuid": "a" + } + ], + "analysis": "1", + "date": "2017-12-31", + "distribution": "1", + "info": "This is a test", + "threat_level_id": "1" + } +} diff --git a/tests/test_mispevent.py b/tests/test_mispevent.py index 2cbeebf..919c2cf 100644 --- a/tests/test_mispevent.py +++ b/tests/test_mispevent.py @@ -140,6 +140,18 @@ class TestMISPEvent(unittest.TestCase): ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + def test_obj_default_values(self): + self.init_event() + self.mispevent.add_object(name='whois', strict=True) + self.mispevent.objects[0].add_attribute('registrar', value='registar.example.com') + self.mispevent.objects[0].add_attribute('domain', value='domain.example.com') + self.mispevent.objects[0].add_attribute('nameserver', value='ns1.example.com') + self.mispevent.objects[0].add_attribute('nameserver', value='ns2.example.com', disable_correlation=False, to_ids=True, category='External analysis') + self.mispevent.objects[0].uuid = 'a' + with open('tests/mispevent_testfiles/def_param.json', 'r') as f: + ref_json = json.load(f) + self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + if __name__ == '__main__': unittest.main() From 8ce7a497af4a63dc42cce703be0ab9ffa0590089 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 5 Jan 2018 11:38:10 +0100 Subject: [PATCH 105/125] fix: Forgotten test files in last commit... --- tests/mispevent_testfiles/event_obj_attr_tag.json | 2 +- tests/mispevent_testfiles/event_obj_def_param.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/mispevent_testfiles/event_obj_attr_tag.json b/tests/mispevent_testfiles/event_obj_attr_tag.json index 42de42e..0981366 100644 --- a/tests/mispevent_testfiles/event_obj_attr_tag.json +++ b/tests/mispevent_testfiles/event_obj_attr_tag.json @@ -10,7 +10,7 @@ } ], "category": "Payload delivery", - "disable_correlation": false, + "disable_correlation": true, "object_relation": "filename", "to_ids": true, "type": "filename", diff --git a/tests/mispevent_testfiles/event_obj_def_param.json b/tests/mispevent_testfiles/event_obj_def_param.json index 56bb0c5..1a06955 100644 --- a/tests/mispevent_testfiles/event_obj_def_param.json +++ b/tests/mispevent_testfiles/event_obj_def_param.json @@ -10,7 +10,7 @@ } ], "category": "Payload delivery", - "disable_correlation": false, + "disable_correlation": true, "object_relation": "filename", "to_ids": true, "type": "filename", @@ -35,7 +35,7 @@ } ], "category": "Payload delivery", - "disable_correlation": false, + "disable_correlation": true, "object_relation": "filename", "to_ids": true, "type": "filename", From a01e7f5e27d579e83498180f84b5d7bcce7216e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 5 Jan 2018 19:17:25 +0100 Subject: [PATCH 106/125] fix: edited method works as expected, add tests. --- pymisp/abstract.py | 7 +- pymisp/mispevent.py | 2 +- tests/mispevent_testfiles/event_tags.json | 20 + tests/mispevent_testfiles/existing_event.json | 16 +- .../existing_event_edited.json | 4575 +++++++++++++++++ tests/test_mispevent.py | 83 +- 6 files changed, 4690 insertions(+), 13 deletions(-) create mode 100644 tests/mispevent_testfiles/event_tags.json create mode 100644 tests/mispevent_testfiles/existing_event_edited.json diff --git a/pymisp/abstract.py b/pymisp/abstract.py index 668a17f..702df34 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -145,10 +145,11 @@ class AbstractMISP(collections.MutableMapping): for p in self.properties: if self.__edited: break - if isinstance(p, AbstractMISP) and p.edited: + val = getattr(self, p) + if isinstance(val, AbstractMISP) and val.edited: self.__edited = True - elif isinstance(p, list) and all(isinstance(a, AbstractMISP) for a in p): - if any(a.edited for a in p): + elif isinstance(val, list) and all(isinstance(a, AbstractMISP) for a in val): + if any(a.edited for a in val): self.__edited = True return self.__edited diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 53a36bd..b44bdaf 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -667,7 +667,7 @@ class MISPEvent(AbstractMISP): def get_object_by_id(self, object_id): """Get an object by ID (the ID is the one set by the server when creating the new object)""" for obj in self.objects: - if hasattr(obj, 'id') and obj.id == object_id: + if hasattr(obj, 'id') and int(obj.id) == int(object_id): return obj raise InvalidMISPObject('Object with {} does not exists in ths event'.format(object_id)) diff --git a/tests/mispevent_testfiles/event_tags.json b/tests/mispevent_testfiles/event_tags.json new file mode 100644 index 0000000..b099b7b --- /dev/null +++ b/tests/mispevent_testfiles/event_tags.json @@ -0,0 +1,20 @@ +{ + "Event": { + "Tag": [ + { + "name": "bar" + }, + { + "name": "baz" + }, + { + "name": "foo" + } + ], + "analysis": "1", + "date": "2017-12-31", + "distribution": "1", + "info": "This is a test", + "threat_level_id": "1" + } +} diff --git a/tests/mispevent_testfiles/existing_event.json b/tests/mispevent_testfiles/existing_event.json index d140132..5f8eac6 100644 --- a/tests/mispevent_testfiles/existing_event.json +++ b/tests/mispevent_testfiles/existing_event.json @@ -67,7 +67,7 @@ "to_ids": false, "type": "text", "uuid": "5a3c2fee-7c8c-438a-8f7f-465402de0b81", - "value": "The Sednit group \u2014 also known as Strontium, APT28, Fancy Bear or Sofacy\u2009\u2014\u2009is a group of attackers operating since 2004, if not earlier, and whose main objective is to steal confidential information from specific targets.\r\n\r\nThis article is a follow-up to ESET\u2019s presentation at BlueHat in November 2017. Late in 2016 we published a white paper covering Sednit activity between 2014 and 2016. Since then, we have continued to actively track Sednit\u2019s operations, and today we are publishing a brief overview of what our tracking uncovered in terms of the group\u2019s activities and updates to their toolset. The first section covers the update of their attack methodology: namely, the ways in which this group tries to compromise their targets systems. The second section covers the evolution of their tools, with a particular emphasis on a detailed analysis of a new version of their flagship malware: Xagent." + "value": "The Sednit group — also known as Strontium, APT28, Fancy Bear or Sofacy — is a group of attackers operating since 2004, if not earlier, and whose main objective is to steal confidential information from specific targets.\r\n\r\nThis article is a follow-up to ESET’s presentation at BlueHat in November 2017. Late in 2016 we published a white paper covering Sednit activity between 2014 and 2016. Since then, we have continued to actively track Sednit’s operations, and today we are publishing a brief overview of what our tracking uncovered in terms of the group’s activities and updates to their toolset. The first section covers the update of their attack methodology: namely, the ways in which this group tries to compromise their targets systems. The second section covers the evolution of their tools, with a particular emphasis on a detailed analysis of a new version of their flagship malware: Xagent." }, { "category": "Network activity", @@ -406,7 +406,7 @@ "Timo Steffens", "Christophe Vandeplas" ], - "description": "This backdoor component is known to have a modular structure featuring various espionage functionalities, such as key-logging, screen grabbing and file exfiltration. This component is available for Osx, Windows, Linux and iOS operating systems.\n\nXagent is a modular backdoor with spying functionalities such as keystroke logging and file exfiltration. Xagent is the group\u2019s flagship backdoor and heavily used in their operations. Early versions for Linux and Windows were seen years ago, then in 2015 an iOS version came out. One year later, an Android version was discovered and finally, in the beginning of 2017, an Xagent sample for OS X was described.", + "description": "This backdoor component is known to have a modular structure featuring various espionage functionalities, such as key-logging, screen grabbing and file exfiltration. This component is available for Osx, Windows, Linux and iOS operating systems.\n\nXagent is a modular backdoor with spying functionalities such as keystroke logging and file exfiltration. Xagent is the group’s flagship backdoor and heavily used in their operations. Early versions for Linux and Windows were seen years ago, then in 2015 an iOS version came out. One year later, an Android version was discovered and finally, in the beginning of 2017, an Xagent sample for OS X was described.", "galaxy_id": "367", "id": "46669", "meta": { @@ -1276,7 +1276,7 @@ "to_ids": true, "type": "filename", "uuid": "5a3cd87d-fa9c-41aa-897f-49a5950d210f", - "value": "Trump\u2019s_Attack_on_Syria_English.docx" + "value": "Trump’s_Attack_on_Syria_English.docx" }, { "category": "Payload delivery", @@ -3920,7 +3920,7 @@ "date": "2017-12-14", "distribution": "3", "id": "9616", - "info": "OSINT - Attackers Deploy New ICS Attack Framework \u201cTRITON\u201d and Cause Operational Disruption to Critical Infrastructure", + "info": "OSINT - Attackers Deploy New ICS Attack Framework “TRITON” and Cause Operational Disruption to Critical Infrastructure", "org_id": "2", "orgc_id": "2", "published": false, @@ -4020,7 +4020,7 @@ "date": "2017-10-23", "distribution": "3", "id": "9208", - "info": "Talos: \u201cCyber Conflict\u201d Decoy Document Used In Real Cyber Conflict", + "info": "Talos: “Cyber Conflict” Decoy Document Used In Real Cyber Conflict", "org_id": "291", "orgc_id": "291", "published": true, @@ -4095,7 +4095,7 @@ "date": "2017-05-11", "distribution": "3", "id": "7820", - "info": "APT28-Sednit adds two zero-day exploits using \u2018Trump\u2019s attack on Syria\u2019 as a decoy", + "info": "APT28-Sednit adds two zero-day exploits using ‘Trump’s attack on Syria’ as a decoy", "org_id": "277", "orgc_id": "277", "published": true, @@ -4145,7 +4145,7 @@ "date": "2016-12-29", "distribution": "3", "id": "5667", - "info": "OSINT - GRIZZLY STEPPE \u2013 Russian Malicious Cyber Activity", + "info": "OSINT - GRIZZLY STEPPE – Russian Malicious Cyber Activity", "org_id": "2", "orgc_id": "2", "published": true, @@ -4270,7 +4270,7 @@ "date": "2016-10-17", "distribution": "3", "id": "5165", - "info": "OSINT: \u2018DealersChoice\u2019 is Sofacy\u2019s Flash Player Exploit Platform", + "info": "OSINT: ‘DealersChoice’ is Sofacy’s Flash Player Exploit Platform", "org_id": "278", "orgc_id": "278", "published": true, diff --git a/tests/mispevent_testfiles/existing_event_edited.json b/tests/mispevent_testfiles/existing_event_edited.json new file mode 100644 index 0000000..84c8f8b --- /dev/null +++ b/tests/mispevent_testfiles/existing_event_edited.json @@ -0,0 +1,4575 @@ +{ + "Event": { + "Attribute": [ + { + "Tag": [ + { + "colour": "#00223b", + "exportable": true, + "hide_tag": false, + "id": "101", + "name": "osint:source-type=\"blog-post\"", + "user_id": "0" + }, + { + "colour": "#007cd6", + "exportable": true, + "hide_tag": false, + "id": "618", + "name": "osint:certainty=\"93\"", + "user_id": "0" + } + ], + "category": "External analysis", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188757", + "object_id": "0", + "sharing_group_id": "0", + "timestamp": "1513893921", + "to_ids": false, + "type": "link", + "uuid": "5a3c2fda-78f4-44b7-8366-46da02de0b81", + "value": "https://www.welivesecurity.com/2017/12/21/sednit-update-fancy-bear-spent-year/" + }, + { + "Tag": [ + { + "colour": "#00223b", + "exportable": true, + "hide_tag": false, + "id": "101", + "name": "osint:source-type=\"blog-post\"", + "user_id": "0" + }, + { + "colour": "#007cd6", + "exportable": true, + "hide_tag": false, + "id": "618", + "name": "osint:certainty=\"93\"", + "user_id": "0" + } + ], + "category": "External analysis", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188758", + "object_id": "0", + "sharing_group_id": "0", + "timestamp": "1513893921", + "to_ids": false, + "type": "text", + "uuid": "5a3c2fee-7c8c-438a-8f7f-465402de0b81", + "value": "The Sednit group — also known as Strontium, APT28, Fancy Bear or Sofacy — is a group of attackers operating since 2004, if not earlier, and whose main objective is to steal confidential information from specific targets.\r\n\r\nThis article is a follow-up to ESET’s presentation at BlueHat in November 2017. Late in 2016 we published a white paper covering Sednit activity between 2014 and 2016. Since then, we have continued to actively track Sednit’s operations, and today we are publishing a brief overview of what our tracking uncovered in terms of the group’s activities and updates to their toolset. The first section covers the update of their attack methodology: namely, the ways in which this group tries to compromise their targets systems. The second section covers the evolution of their tools, with a particular emphasis on a detailed analysis of a new version of their flagship malware: Xagent." + }, + { + "category": "Network activity", + "comment": "Xagent Samples", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188759", + "object_id": "0", + "sharing_group_id": "0", + "timestamp": "1513893957", + "to_ids": true, + "type": "domain", + "uuid": "5a3c3045-ab0c-4d38-8efe-459002de0b81", + "value": "movieultimate.com" + }, + { + "category": "Network activity", + "comment": "Xagent Samples", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188760", + "object_id": "0", + "sharing_group_id": "0", + "timestamp": "1513893957", + "to_ids": true, + "type": "domain", + "uuid": "5a3c3045-61dc-495c-ae8a-471e02de0b81", + "value": "meteost.com" + }, + { + "category": "Network activity", + "comment": "Xagent Samples", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188761", + "object_id": "0", + "sharing_group_id": "0", + "timestamp": "1513893957", + "to_ids": true, + "type": "domain", + "uuid": "5a3c3045-e354-4978-a6b4-49ad02de0b81", + "value": "faststoragefiles.org" + }, + { + "category": "Network activity", + "comment": "Xagent Samples", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188762", + "object_id": "0", + "sharing_group_id": "0", + "timestamp": "1513893957", + "to_ids": true, + "type": "domain", + "uuid": "5a3c3045-968c-4572-9f64-491502de0b81", + "value": "nethostnet.com" + }, + { + "category": "Network activity", + "comment": "Xagent Samples", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188763", + "object_id": "0", + "sharing_group_id": "0", + "timestamp": "1513893957", + "to_ids": true, + "type": "domain", + "uuid": "5a3c3045-eb44-433f-a13a-44b902de0b81", + "value": "fsportal.net" + }, + { + "category": "Network activity", + "comment": "Xagent Samples", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188764", + "object_id": "0", + "sharing_group_id": "0", + "timestamp": "1513893957", + "to_ids": true, + "type": "domain", + "uuid": "5a3c3045-6a88-479d-b799-4d3d02de0b81", + "value": "fastdataexchange.org" + }, + { + "category": "Network activity", + "comment": "Xagent Samples", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188765", + "object_id": "0", + "sharing_group_id": "0", + "timestamp": "1513893957", + "to_ids": true, + "type": "domain", + "uuid": "5a3c3045-7480-4831-a5c4-48c802de0b81", + "value": "newfilmts.com" + } + ], + "Galaxy": [ + { + "GalaxyCluster": [ + { + "authors": [ + "Alexandre Dulaunoy", + "Florian Roth", + "Thomas Schreck", + "Timo Steffens", + "Various" + ], + "description": "The Sofacy Group (also known as APT28, Pawn Storm, Fancy Bear and Sednit) is a cyber espionage group believed to have ties to the Russian government. Likely operating since 2007, the group is known to target government, military, and security organizations. It has been characterized as an advanced persistent threat.", + "galaxy_id": "366", + "id": "45563", + "meta": { + "country": [ + "RU" + ], + "refs": [ + "https://en.wikipedia.org/wiki/Sofacy_Group", + "https://aptnotes.malwareconfig.com/web/viewer.html?file=../APTnotes/2014/apt28.pdf", + "http://www.trendmicro.com/cloud-content/us/pdfs/security-intelligence/white-papers/wp-operation-pawn-storm.pdf", + "https://www2.fireeye.com/rs/848-DID-242/images/wp-mandiant-matryoshka-mining.pdf", + "https://www.crowdstrike.com/blog/bears-midst-intrusion-democratic-national-committee/", + "http://researchcenter.paloaltonetworks.com/2016/06/unit42-new-sofacy-attacks-against-us-government-agency/" + ], + "synonyms": [ + "APT 28", + "APT28", + "Pawn Storm", + "Fancy Bear", + "Sednit", + "TsarTeam", + "TG-4127", + "Group-4127", + "STRONTIUM", + "TAG_0700", + "Swallowtail", + "IRON TWILIGHT", + "Group 74" + ] + }, + "source": "MISP Project", + "tag_id": "1100", + "tag_name": "misp-galaxy:threat-actor=\"Sofacy\"", + "type": "threat-actor", + "uuid": "7cdff317-a673-4474-84ec-4f1754947823", + "value": "Sofacy", + "version": "30" + } + ], + "description": "Threat actors are characteristics of malicious actors (or adversaries) representing a cyber attack threat including presumed intent and historically observed behaviour.", + "icon": "user-secret", + "id": "366", + "name": "Threat Actor", + "type": "threat-actor", + "uuid": "698774c7-8022-42c4-917f-8d6e4f06ada3", + "version": "2" + }, + { + "GalaxyCluster": [ + { + "authors": [ + "Kafeine", + "Will Metcalf", + "KahuSecurity" + ], + "description": "Sednit EK is the exploit kit used by APT28", + "galaxy_id": "370", + "id": "38813", + "meta": { + "refs": [ + "http://www.welivesecurity.com/2014/10/08/sednit-espionage-group-now-using-custom-exploit-kit/", + "http://blog.trendmicro.com/trendlabs-security-intelligence/new-adobe-flash-zero-day-used-in-pawn-storm-campaign/" + ], + "status": [ + "Active" + ] + }, + "source": "MISP Project", + "tag_id": "3007", + "tag_name": "misp-galaxy:exploit-kit=\"Sednit EK\"", + "type": "exploit-kit", + "uuid": "454f4e78-bd7c-11e6-a4a6-cec0c932ce01", + "value": "Sednit EK", + "version": "5" + }, + { + "authors": [ + "Kafeine", + "Will Metcalf", + "KahuSecurity" + ], + "description": "DealersChoice is a Flash Player Exploit platform triggered by RTF", + "galaxy_id": "370", + "id": "38805", + "meta": { + "refs": [ + "http://researchcenter.paloaltonetworks.com/2016/10/unit42-dealerschoice-sofacys-flash-player-exploit-platform/", + "http://blog.trendmicro.com/trendlabs-security-intelligence/pawn-storm-ramps-up-spear-phishing-before-zero-days-get-patched/" + ], + "status": [ + "Active" + ], + "synonyms": [ + "Sednit RTF EK" + ] + }, + "source": "MISP Project", + "tag_id": "3015", + "tag_name": "misp-galaxy:exploit-kit=\"DealersChoice\"", + "type": "exploit-kit", + "uuid": "454f4e78-bd7c-11e6-a4a6-cec0c932ce01", + "value": "DealersChoice", + "version": "5" + } + ], + "description": "Exploit-Kit is an enumeration of some exploitation kits used by adversaries. The list includes document, browser and router exploit kits.It's not meant to be totally exhaustive but aim at covering the most seen in the past 5 years", + "icon": "internet-explorer", + "id": "370", + "name": "Exploit-Kit", + "type": "exploit-kit", + "uuid": "6ab240ec-bd79-11e6-a4a6-cec0c932ce01", + "version": "3" + }, + { + "GalaxyCluster": [ + { + "authors": [ + "Alexandre Dulaunoy", + "Florian Roth", + "Timo Steffens", + "Christophe Vandeplas" + ], + "description": "backdoor", + "galaxy_id": "367", + "id": "46592", + "meta": { + "refs": [ + "https://www2.fireeye.com/rs/848-DID-242/images/APT28-Center-of-Storm-2017.pdf" + ], + "synonyms": [ + "Sednit", + "Seduploader", + "JHUHUGIT", + "Sofacy" + ], + "type": [ + "Backdoor" + ] + }, + "source": "MISP Project", + "tag_id": "2215", + "tag_name": "misp-galaxy:tool=\"GAMEFISH\"", + "type": "tool", + "uuid": "0d821b68-9d82-4c6d-86a6-1071a9e0f79f", + "value": "GAMEFISH", + "version": "45" + }, + { + "authors": [ + "Alexandre Dulaunoy", + "Florian Roth", + "Timo Steffens", + "Christophe Vandeplas" + ], + "description": "", + "galaxy_id": "367", + "id": "46670", + "meta": { + "synonyms": [ + "XTunnel" + ] + }, + "source": "MISP Project", + "tag_id": "1012", + "tag_name": "misp-galaxy:tool=\"X-Tunnel\"", + "type": "tool", + "uuid": "0d821b68-9d82-4c6d-86a6-1071a9e0f79f", + "value": "X-Tunnel", + "version": "45" + }, + { + "authors": [ + "Alexandre Dulaunoy", + "Florian Roth", + "Timo Steffens", + "Christophe Vandeplas" + ], + "description": "backdoor used by apt28\n\nSedreco serves as a spying backdoor; its functionalities can be extended with dynamically loaded plugins. It is made up of two distinct components: a dropper and the persistent payload installed by this dropper. We have not seen this component since April 2016.", + "galaxy_id": "367", + "id": "46591", + "meta": { + "possible_issues": [ + "Report tells that is could be Xagent alias (Java Rat)" + ], + "refs": [ + "https://www2.fireeye.com/rs/848-DID-242/images/APT28-Center-of-Storm-2017.pdf" + ], + "synonyms": [ + "Sedreco", + "AZZY", + "ADVSTORESHELL", + "NETUI" + ], + "type": [ + "Backdoor" + ] + }, + "source": "MISP Project", + "tag_id": "3011", + "tag_name": "misp-galaxy:tool=\"EVILTOSS\"", + "type": "tool", + "uuid": "0d821b68-9d82-4c6d-86a6-1071a9e0f79f", + "value": "EVILTOSS", + "version": "45" + }, + { + "authors": [ + "Alexandre Dulaunoy", + "Florian Roth", + "Timo Steffens", + "Christophe Vandeplas" + ], + "description": "This backdoor component is known to have a modular structure featuring various espionage functionalities, such as key-logging, screen grabbing and file exfiltration. This component is available for Osx, Windows, Linux and iOS operating systems.\n\nXagent is a modular backdoor with spying functionalities such as keystroke logging and file exfiltration. Xagent is the group’s flagship backdoor and heavily used in their operations. Early versions for Linux and Windows were seen years ago, then in 2015 an iOS version came out. One year later, an Android version was discovered and finally, in the beginning of 2017, an Xagent sample for OS X was described.", + "galaxy_id": "367", + "id": "46669", + "meta": { + "refs": [ + "http://blog.trendmicro.com/trendlabs-security-intelligence/pawn-storm-update-ios-espionage-app-found/", + "https://app.box.com/s/l7n781ig6n8wlf1aff5hgwbh4qoi5jqq", + "https://www.welivesecurity.com/2017/12/21/sednit-update-fancy-bear-spent-year/" + ], + "synonyms": [ + "XAgent" + ], + "type": [ + "Backdoor" + ] + }, + "source": "MISP Project", + "tag_id": "1011", + "tag_name": "misp-galaxy:tool=\"X-Agent\"", + "type": "tool", + "uuid": "0d821b68-9d82-4c6d-86a6-1071a9e0f79f", + "value": "X-Agent", + "version": "45" + } + ], + "description": "Threat actors tools is an enumeration of tools used by adversaries. The list includes malware but also common software regularly used by the adversaries.", + "icon": "optin-monster", + "id": "367", + "name": "Tool", + "type": "tool", + "uuid": "9b8037f7-bc8f-4de1-a797-37266619bc0b", + "version": "2" + }, + { + "GalaxyCluster": [ + { + "authors": [ + "MITRE" + ], + "description": "JHUHUGIT is malware used by APT28. It is based on Carberp source code and serves as reconnaissance malware.[[Citation: Kaspersky Sofacy]][[Citation: F-Secure Sofacy 2015]][[Citation: ESET Sednit Part 1]][[Citation: FireEye APT28 January 2017]]\n\nAliases: JHUHUGIT, Seduploader, JKEYSKW, Sednit, GAMEFISH", + "galaxy_id": "365", + "id": "41618", + "meta": { + "refs": [ + "https://attack.mitre.org/wiki/Software/S0044", + "http://www.welivesecurity.com/wp-content/uploads/2016/10/eset-sednit-part1.pdf", + "https://www2.fireeye.com/rs/848-DID-242/images/APT28-Center-of-Storm-2017.pdf", + "https://labsblog.f-secure.com/2015/09/08/sofacy-recycles-carberp-and-metasploit-code/", + "https://securelist.com/blog/research/72924/sofacy-apt-hits-high-profile-targets-with-updated-toolset/" + ], + "synonyms": [ + "JHUHUGIT", + "Seduploader", + "JKEYSKW", + "Sednit", + "GAMEFISH" + ], + "uuid": [ + "8ae43c46-57ef-47d5-a77a-eebb35628db2" + ] + }, + "source": "https://github.com/mitre/cti", + "tag_id": "3008", + "tag_name": "misp-galaxy:mitre-malware=\"JHUHUGIT\"", + "type": "mitre-malware", + "uuid": "d752161c-78f6-11e7-a0ea-bfa79b407ce4", + "value": "JHUHUGIT", + "version": "4" + }, + { + "authors": [ + "MITRE" + ], + "description": "XTunnel a VPN-like network proxy tool that can relay traffic between a C2 server and a victim. It was first seen in May 2013 and reportedly used by APT28 during the compromise of the Democratic National Committee.[[Citation: Crowdstrike DNC June 2016]][[Citation: Invincea XTunnel]][[Citation: ESET Sednit Part 2]]\n\nAliases: XTunnel, X-Tunnel, XAPS", + "galaxy_id": "365", + "id": "41543", + "meta": { + "refs": [ + "https://attack.mitre.org/wiki/Software/S0117", + "http://www.welivesecurity.com/wp-content/uploads/2016/10/eset-sednit-part-2.pdf", + "https://www.invincea.com/2016/07/tunnel-of-gov-dnc-hack-and-the-russian-xtunnel/", + "https://www.crowdstrike.com/blog/bears-midst-intrusion-democratic-national-committee/" + ], + "synonyms": [ + "XTunnel", + "X-Tunnel", + "XAPS" + ], + "uuid": [ + "7343e208-7cab-45f2-a47b-41ba5e2f0fab" + ] + }, + "source": "https://github.com/mitre/cti", + "tag_id": "3009", + "tag_name": "misp-galaxy:mitre-malware=\"XTunnel\"", + "type": "mitre-malware", + "uuid": "d752161c-78f6-11e7-a0ea-bfa79b407ce4", + "value": "XTunnel", + "version": "4" + }, + { + "authors": [ + "MITRE" + ], + "description": "ADVSTORESHELL is a spying backdoor that has been used by APT28 from at least 2012 to 2016. It is generally used for long-term espionage and is deployed on targets deemed interesting after a reconnaissance phase.[[Citation: Kaspersky Sofacy]][[Citation: ESET Sednit Part 2]]\n\nAliases: ADVSTORESHELL, NETUI, EVILTOSS, AZZY, Sedreco", + "galaxy_id": "365", + "id": "41582", + "meta": { + "refs": [ + "https://attack.mitre.org/wiki/Software/S0045", + "http://www.welivesecurity.com/wp-content/uploads/2016/10/eset-sednit-part-2.pdf", + "https://securelist.com/blog/research/72924/sofacy-apt-hits-high-profile-targets-with-updated-toolset/" + ], + "synonyms": [ + "ADVSTORESHELL", + "NETUI", + "EVILTOSS", + "AZZY", + "Sedreco" + ], + "uuid": [ + "fb575479-14ef-41e9-bfab-0b7cf10bec73" + ] + }, + "source": "https://github.com/mitre/cti", + "tag_id": "3010", + "tag_name": "misp-galaxy:mitre-malware=\"ADVSTORESHELL\"", + "type": "mitre-malware", + "uuid": "d752161c-78f6-11e7-a0ea-bfa79b407ce4", + "value": "ADVSTORESHELL", + "version": "4" + }, + { + "authors": [ + "MITRE" + ], + "description": "USBStealer is malware that has used by APT28 since at least 2005 to extract information from air-gapped networks. It does not have the capability to communicate over the Internet and has been used in conjunction with ADVSTORESHELL.[[Citation: ESET Sednit USBStealer 2014]][[Citation: Kaspersky Sofacy]]\n\nAliases: USBStealer, USB Stealer, Win32/USBStealer", + "galaxy_id": "365", + "id": "41549", + "meta": { + "refs": [ + "https://attack.mitre.org/wiki/Software/S0136", + "http://www.welivesecurity.com/2014/11/11/sednit-espionage-group-attacking-air-gapped-networks/", + "https://securelist.com/blog/research/72924/sofacy-apt-hits-high-profile-targets-with-updated-toolset/" + ], + "synonyms": [ + "USBStealer", + "USB Stealer", + "Win32/USBStealer" + ], + "uuid": [ + "af2ad3b7-ab6a-4807-91fd-51bcaff9acbb" + ] + }, + "source": "https://github.com/mitre/cti", + "tag_id": "3012", + "tag_name": "misp-galaxy:mitre-malware=\"USBStealer\"", + "type": "mitre-malware", + "uuid": "d752161c-78f6-11e7-a0ea-bfa79b407ce4", + "value": "USBStealer", + "version": "4" + }, + { + "authors": [ + "MITRE" + ], + "description": "is a trojan that has been used by APT28 on OS X and appears to be a port of their standard CHOPSTICK or XAgent trojan.[[Citation: XAgentOSX]]", + "galaxy_id": "365", + "id": "41551", + "meta": { + "refs": [ + "https://attack.mitre.org/wiki/Software/S0161", + "https://researchcenter.paloaltonetworks.com/2017/02/unit42-xagentosx-sofacys-xagent-macos-tool/" + ], + "uuid": [ + "5930509b-7793-4db9-bdfc-4edda7709d0d" + ] + }, + "source": "https://github.com/mitre/cti", + "tag_id": "3013", + "tag_name": "misp-galaxy:mitre-malware=\"XAgentOSX\"", + "type": "mitre-malware", + "uuid": "d752161c-78f6-11e7-a0ea-bfa79b407ce4", + "value": "XAgentOSX", + "version": "4" + }, + { + "authors": [ + "MITRE" + ], + "description": "CHOPSTICK is malware family of modular backdoors used by APT28. It has been used from at least November 2012 to August 2016 and is usually dropped on victims as second-stage malware, though it has been used as first-stage malware in several cases.[[Citation: FireEye APT28]][[Citation: ESET Sednit Part 2]][[Citation: FireEye APT28 January 2017]]\n\nAliases: CHOPSTICK, SPLM, Xagent, X-Agent, webhp", + "galaxy_id": "365", + "id": "41559", + "meta": { + "refs": [ + "https://attack.mitre.org/wiki/Software/S0023", + "https://www2.fireeye.com/rs/848-DID-242/images/APT28-Center-of-Storm-2017.pdf", + "http://www.welivesecurity.com/wp-content/uploads/2016/10/eset-sednit-part-2.pdf", + "https://www.fireeye.com/content/dam/fireeye-www/global/en/current-threats/pdfs/rpt-apt28.pdf" + ], + "synonyms": [ + "CHOPSTICK", + "SPLM", + "Xagent", + "X-Agent", + "webhp" + ], + "uuid": [ + "ccd61dfc-b03f-4689-8c18-7c97eab08472" + ] + }, + "source": "https://github.com/mitre/cti", + "tag_id": "3014", + "tag_name": "misp-galaxy:mitre-malware=\"CHOPSTICK\"", + "type": "mitre-malware", + "uuid": "d752161c-78f6-11e7-a0ea-bfa79b407ce4", + "value": "CHOPSTICK", + "version": "4" + }, + { + "authors": [ + "MITRE" + ], + "description": "Downdelph is a first-stage downloader written in Delphi that has been used by APT28 in rare instances between 2013 and 2015.[[Citation: ESET Sednit Part 3]]\n\nAliases: Downdelph, Delphacy", + "galaxy_id": "365", + "id": "41504", + "meta": { + "refs": [ + "https://attack.mitre.org/wiki/Software/S0134", + "http://www.welivesecurity.com/wp-content/uploads/2016/10/eset-sednit-part3.pdf" + ], + "synonyms": [ + "Downdelph", + "Delphacy" + ], + "uuid": [ + "08d20cd2-f084-45ee-8558-fa6ef5a18519" + ] + }, + "source": "https://github.com/mitre/cti", + "tag_id": "3016", + "tag_name": "misp-galaxy:mitre-malware=\"Downdelph\"", + "type": "mitre-malware", + "uuid": "d752161c-78f6-11e7-a0ea-bfa79b407ce4", + "value": "Downdelph", + "version": "4" + } + ], + "description": "Name of ATT&CK software", + "icon": "optin-monster", + "id": "365", + "name": "Malware", + "type": "mitre-malware", + "uuid": "d752161c-78f6-11e7-a0ea-bfa79b407ce4", + "version": "4" + } + ], + "Object": [ + { + "Attribute": [ + { + "Tag": [ + { + "name": "blah" + } + ], + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188944", + "object_id": "1555", + "object_relation": "filename", + "sharing_group_id": "0", + "to_ids": true, + "type": "filename", + "uuid": "5a3cd5b6-2850-435f-bd0d-4c62950d210f", + "value": "Bulletin.doc" + }, + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188945", + "object_id": "1555", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513936310", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd5b6-78a8-4e47-8333-4c62950d210f", + "value": "68064fc152e23d56e541714af52651cb4ba81aaf" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188946", + "object_id": "1555", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513936310", + "to_ids": false, + "type": "text", + "uuid": "5a3cd5b6-23d8-43ba-8518-4c62950d210f", + "value": "Malicious" + } + ], + "comment": "Win32/Sednit.AX", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1555", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "uuid": "5a3cd5b6-9568-4342-b2ab-4c62950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188947", + "object_id": "1556", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513936388", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd604-748c-4fc0-88bf-c170950d210f", + "value": "f3805382ae2e23ff1147301d131a06e00e4ff75f" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188948", + "object_id": "1556", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513936388", + "to_ids": false, + "type": "text", + "uuid": "5a3cd604-6668-4469-a1c0-c170950d210f", + "value": "Malicious" + } + ], + "comment": "Win32/Exploit.CVE-2016-4117.A", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1556", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513936388", + "uuid": "5a3cd604-e11c-4de5-bbbf-c170950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188949", + "object_id": "1557", + "object_relation": "filename", + "sharing_group_id": "0", + "timestamp": "1513936531", + "to_ids": true, + "type": "filename", + "uuid": "5a3cd693-dc40-445d-a4d7-4ae0950d210f", + "value": "OC_PSO_2017.doc" + }, + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188950", + "object_id": "1557", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513936531", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd693-8ffc-4d95-b522-4e84950d210f", + "value": "512bdfe937314ac3f195c462c395feeb36932971" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188951", + "object_id": "1557", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513936531", + "to_ids": false, + "type": "text", + "uuid": "5a3cd693-a8f0-4aea-a834-4097950d210f", + "value": "Malicious" + } + ], + "comment": "Win32/Exploit.Agent.NUB", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1557", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513936531", + "uuid": "5a3cd693-fd9c-4fcf-b69a-439c950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188952", + "object_id": "1558", + "object_relation": "filename", + "sharing_group_id": "0", + "timestamp": "1513936578", + "to_ids": true, + "type": "filename", + "uuid": "5a3cd6c2-d31c-40cc-bcc1-4458950d210f", + "value": "NASAMS.doc" + }, + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188953", + "object_id": "1558", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513936578", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd6c2-6a54-4b4c-8748-4c84950d210f", + "value": "30b3e8c0f3f3cf200daa21c267ffab3cad64e68b" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188954", + "object_id": "1558", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513936578", + "to_ids": false, + "type": "text", + "uuid": "5a3cd6c2-1c68-45de-8325-464a950d210f", + "value": "Malicious" + } + ], + "comment": "Win32/Exploit.Agent.NTR", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1558", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513936578", + "uuid": "5a3cd6c2-d290-4787-910f-4e6d950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188955", + "object_id": "1559", + "object_relation": "filename", + "sharing_group_id": "0", + "timestamp": "1513936718", + "to_ids": true, + "type": "filename", + "uuid": "5a3cd74e-584c-45b9-8557-486d950d210f", + "value": "Programm_Details.doc" + }, + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188956", + "object_id": "1559", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513936718", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd74e-f334-4e6b-b37f-462f950d210f", + "value": "4173b29a251cd9c1cab135f67cb60acab4ace0c5" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188957", + "object_id": "1559", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513936718", + "to_ids": false, + "type": "text", + "uuid": "5a3cd74e-5900-4fbf-85c6-4c81950d210f", + "value": "Malicious" + } + ], + "comment": "Win32/Exploit.Agent.NTO", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1559", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513936718", + "uuid": "5a3cd74e-1504-40ff-9a28-4501950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188958", + "object_id": "1560", + "object_relation": "filename", + "sharing_group_id": "0", + "timestamp": "1513936757", + "to_ids": true, + "type": "filename", + "uuid": "5a3cd775-e8f4-465a-aca2-4c5a950d210f", + "value": "Operation_in_Mosul.rtf" + }, + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188959", + "object_id": "1560", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513936757", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd775-1190-4db7-961a-4c5a950d210f", + "value": "12a37cfdd3f3671074dd5b0f354269cec028fb52" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188960", + "object_id": "1560", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513936757", + "to_ids": false, + "type": "text", + "uuid": "5a3cd775-fa5c-4453-bcb0-4c5a950d210f", + "value": "Malicious" + } + ], + "comment": "Win32/Exploit.Agent.NTR", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1560", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513936757", + "uuid": "5a3cd775-e4cc-44bb-89b6-4c5a950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188961", + "object_id": "1561", + "object_relation": "filename", + "sharing_group_id": "0", + "timestamp": "1513936943", + "to_ids": true, + "type": "filename", + "uuid": "5a3cd82f-b918-4520-ba8b-5165950d210f", + "value": "ARM-NATO_ENGLISH_30_NOV_2016.doc" + }, + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188962", + "object_id": "1561", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513936943", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd82f-cae4-4209-9338-5165950d210f", + "value": "15201766bd964b7c405aeb11db81457220c31e46" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188963", + "object_id": "1561", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513936943", + "to_ids": false, + "type": "text", + "uuid": "5a3cd82f-d91c-43af-8262-5165950d210f", + "value": "Malicious" + } + ], + "comment": "SWF/Agent.L", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1561", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513936943", + "uuid": "5a3cd82f-2788-4561-bbeb-5165950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188964", + "object_id": "1562", + "object_relation": "filename", + "sharing_group_id": "0", + "timestamp": "1513936967", + "to_ids": true, + "type": "filename", + "uuid": "5a3cd847-0aa0-4b5c-aa30-5165950d210f", + "value": "Olympic-Agenda-2020-20-20-Recommendations.doc" + }, + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188965", + "object_id": "1562", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513936967", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd847-593c-4985-8756-5165950d210f", + "value": "8078e411fbe33864dfd8f87ad5105cc1fd26d62e" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188966", + "object_id": "1562", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513936967", + "to_ids": false, + "type": "text", + "uuid": "5a3cd847-1324-4fad-af60-5165950d210f", + "value": "Malicious" + } + ], + "comment": "Win32/Exploit.Agent.BL", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1562", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513936967", + "uuid": "5a3cd847-b5a0-42f7-ac4b-5165950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188967", + "object_id": "1563", + "object_relation": "filename", + "sharing_group_id": "0", + "timestamp": "1513936993", + "to_ids": true, + "type": "filename", + "uuid": "5a3cd861-9350-40c1-ac29-4771950d210f", + "value": "Merry_Christmas!.docx" + }, + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188968", + "object_id": "1563", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513936993", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd861-18ac-4cf0-b96f-4986950d210f", + "value": "33447383379ca99083442b852589111296f0c603" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188969", + "object_id": "1563", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513936993", + "to_ids": false, + "type": "text", + "uuid": "5a3cd861-cfbc-4096-baae-40e2950d210f", + "value": "Malicious" + } + ], + "comment": "Win32/Exploit.Agent.NUG", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1563", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513936993", + "uuid": "5a3cd861-65c0-4b69-9429-4f37950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188970", + "object_id": "1564", + "object_relation": "filename", + "sharing_group_id": "0", + "timestamp": "1513937021", + "to_ids": true, + "type": "filename", + "uuid": "5a3cd87d-fa9c-41aa-897f-49a5950d210f", + "value": "Trump’s_Attack_on_Syria_English.docx" + }, + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188971", + "object_id": "1564", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937021", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd87d-c630-4487-8336-4615950d210f", + "value": "d5235d136cfcadbef431eea7253d80bde414db9d" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188972", + "object_id": "1564", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937021", + "to_ids": false, + "type": "text", + "uuid": "5a3cd87d-8c98-4660-9026-44de950d210f", + "value": "Malicious" + } + ], + "comment": "Win32/Exploit.Agent.NWZ", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1564", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513937021", + "uuid": "5a3cd87d-f514-4071-a5f7-4ec2950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188973", + "object_id": "1565", + "object_relation": "filename", + "sharing_group_id": "0", + "timestamp": "1513937047", + "to_ids": true, + "type": "filename", + "uuid": "5a3cd897-4cc0-48b0-bb2c-461f950d210f", + "value": "Hotel_Reservation_Form.doc" + }, + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188974", + "object_id": "1565", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937047", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd897-fa64-466c-9421-49c5950d210f", + "value": "f293a2bfb728060c54efeeb03c5323893b5c80df" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188975", + "object_id": "1565", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937047", + "to_ids": false, + "type": "text", + "uuid": "5a3cd897-f020-44cf-8dfc-4225950d210f", + "value": "Malicious" + } + ], + "comment": "Win32/Sednit.BN", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1565", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513937046", + "uuid": "5a3cd896-f6cc-4e52-bcb2-442c950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188976", + "object_id": "1566", + "object_relation": "filename", + "sharing_group_id": "0", + "timestamp": "1513937070", + "to_ids": true, + "type": "filename", + "uuid": "5a3cd8ae-7194-48fd-810e-4c5a950d210f", + "value": "SB_Doc_2017-3_Implementation_of_Key_Taskings_and_Next_Steps.doc" + }, + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188977", + "object_id": "1566", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937071", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd8af-f39c-443c-bcf1-4c5a950d210f", + "value": "bb10ed5d59672fbc6178e35d0feac0562513e9f0" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188978", + "object_id": "1566", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937071", + "to_ids": false, + "type": "text", + "uuid": "5a3cd8af-b3ec-478a-b585-4c5a950d210f", + "value": "Malicious" + } + ], + "comment": "Win32/Sednit.BN", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1566", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513937070", + "uuid": "5a3cd8ae-54d0-46bb-adbb-4c5a950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188979", + "object_id": "1567", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937083", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd8bb-74d8-4d19-ae08-4043950d210f", + "value": "4873bafe44cff06845faa0ce7c270c4ce3c9f7b9" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188980", + "object_id": "1567", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937083", + "to_ids": false, + "type": "text", + "uuid": "5a3cd8bb-77bc-4cc4-887f-429d950d210f", + "value": "Malicious" + } + ], + "comment": "", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1567", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513937083", + "uuid": "5a3cd8bb-a704-4f1d-a235-444e950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188981", + "object_id": "1568", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937097", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd8c9-4d2c-4145-a637-4f13950d210f", + "value": "169c8f3e3d22e192c108bc95164d362ce5437465" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188982", + "object_id": "1568", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937097", + "to_ids": false, + "type": "text", + "uuid": "5a3cd8c9-7ff0-42f7-ae80-4eb6950d210f", + "value": "Malicious" + } + ], + "comment": "", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1568", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513937097", + "uuid": "5a3cd8c9-6568-406a-853c-4862950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188983", + "object_id": "1569", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937116", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd8dc-48c0-4ea0-a67d-4734950d210f", + "value": "cc7607015cd7a1a4452acd3d87adabdd7e005bd7" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188984", + "object_id": "1569", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937116", + "to_ids": false, + "type": "text", + "uuid": "5a3cd8dc-9ed8-4a4d-9ceb-4daa950d210f", + "value": "Malicious" + } + ], + "comment": "Win32/Sednit.BN", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1569", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513937115", + "uuid": "5a3cd8db-2838-4466-a986-4afb950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188985", + "object_id": "1570", + "object_relation": "filename", + "sharing_group_id": "0", + "timestamp": "1513937147", + "to_ids": true, + "type": "filename", + "uuid": "5a3cd8fb-1efc-4059-ae7a-42f5950d210f", + "value": "Caucasian_Eagle_ENG.docx" + }, + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188986", + "object_id": "1570", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937147", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd8fb-9cec-4a30-8b2f-4441950d210f", + "value": "5d2c7d87995cc5b8184baba2c7a1900a48b2f42d" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188987", + "object_id": "1570", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937147", + "to_ids": false, + "type": "text", + "uuid": "5a3cd8fb-e52c-489b-8da5-43d1950d210f", + "value": "Malicious" + } + ], + "comment": "Win32/Exploit.Agent.NTM", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1570", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513937147", + "uuid": "5a3cd8fb-cd14-4b00-9710-430c950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188988", + "object_id": "1571", + "object_relation": "filename", + "sharing_group_id": "0", + "timestamp": "1513937166", + "to_ids": true, + "type": "filename", + "uuid": "5a3cd90e-5eb4-4069-b160-5276950d210f", + "value": "World War3.docx" + }, + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188989", + "object_id": "1571", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937166", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd90e-6d2c-4ffc-a699-5276950d210f", + "value": "7aada8bcc0d1ab8ffb1f0fae4757789c6f5546a3" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188990", + "object_id": "1571", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937166", + "to_ids": false, + "type": "text", + "uuid": "5a3cd90e-28e8-410e-8033-5276950d210f", + "value": "Malicious" + } + ], + "comment": "SWF/Exploit.CVE-2017-11292.A", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1571", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513937166", + "uuid": "5a3cd90e-538c-4b7e-95dc-5276950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188991", + "object_id": "1572", + "object_relation": "filename", + "sharing_group_id": "0", + "timestamp": "1513937191", + "to_ids": true, + "type": "filename", + "uuid": "5a3cd927-e810-4d22-a0e4-4057950d210f", + "value": "SaberGuardian2017.docx" + }, + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188992", + "object_id": "1572", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937191", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd927-f284-43b9-83d1-473b950d210f", + "value": "68c2809560c7623d2307d8797691abf3eafe319a" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188993", + "object_id": "1572", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937191", + "to_ids": false, + "type": "text", + "uuid": "5a3cd927-b844-49f2-a1a9-4c85950d210f", + "value": "Malicious" + } + ], + "comment": "VBA/DDE.E", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1572", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513937191", + "uuid": "5a3cd927-e410-489c-abfc-4b63950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188994", + "object_id": "1573", + "object_relation": "filename", + "sharing_group_id": "0", + "timestamp": "1513937212", + "to_ids": true, + "type": "filename", + "uuid": "5a3cd93c-2438-4dda-823e-463d950d210f", + "value": "IsisAttackInNewYork.docx" + }, + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188995", + "object_id": "1573", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937212", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cd93c-1ef0-4d81-9476-4655950d210f", + "value": "1c6c700ceebfbe799e115582665105caa03c5c9e" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188996", + "object_id": "1573", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937212", + "to_ids": false, + "type": "text", + "uuid": "5a3cd93c-949c-40ac-9094-4a4a950d210f", + "value": "Malicious" + } + ], + "comment": "VBA/DDE.L", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1573", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513937212", + "uuid": "5a3cd93c-716c-4918-a00f-4671950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188997", + "object_id": "1574", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937559", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cda97-7e58-4642-aaf5-c5ed950d210f", + "value": "6f0fc0ebba3e4c8b26a69cdf519edf8d1aa2f4bb" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1188998", + "object_id": "1574", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937559", + "to_ids": false, + "type": "text", + "uuid": "5a3cda97-6020-423d-9d23-c5ed950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Attribute": { + "category": "Network activity", + "distribution": "5", + "sharing_group_id": "0", + "to_ids": true, + "type": "domain", + "uuid": "5a3c3045-ab0c-4d38-8efe-459002de0b81", + "value": "movieultimate.com" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "159", + "object_id": "1574", + "object_uuid": "5a3cda96-85c4-45a1-82ea-c5ed950d210f", + "referenced_id": "1188759", + "referenced_type": "0", + "referenced_uuid": "5a3c3045-ab0c-4d38-8efe-459002de0b81", + "relationship_type": "communicates-with", + "timestamp": "1513937826", + "uuid": "5a3cdba2-2fdc-4f9a-a4eb-4dae950d210f" + } + ], + "comment": "Win64/Sednit.Z", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1574", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513937826", + "uuid": "5a3cda96-85c4-45a1-82ea-c5ed950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1188999", + "object_id": "1575", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937864", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cdbc8-0aac-4d8a-8c1f-4c5a950d210f", + "value": "e19f753e514f6adec8f81bcdefb9117979e69627" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189000", + "object_id": "1575", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937864", + "to_ids": false, + "type": "text", + "uuid": "5a3cdbc8-e204-4606-b9ea-4c5a950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Attribute": { + "category": "Network activity", + "distribution": "5", + "sharing_group_id": "0", + "to_ids": true, + "type": "domain", + "uuid": "5a3c3045-61dc-495c-ae8a-471e02de0b81", + "value": "meteost.com" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "160", + "object_id": "1575", + "object_uuid": "5a3cdbc7-dbec-4b8c-8ba3-4c5a950d210f", + "referenced_id": "1188760", + "referenced_type": "0", + "referenced_uuid": "5a3c3045-61dc-495c-ae8a-471e02de0b81", + "relationship_type": "communicates-with", + "timestamp": "1513938091", + "uuid": "5a3cdcab-8200-4c65-868e-42a9950d210f" + } + ], + "comment": "Win64/Sednit.Z", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1575", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513938091", + "uuid": "5a3cdbc7-dbec-4b8c-8ba3-4c5a950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189001", + "object_id": "1576", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937910", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cdbf6-eca0-4c09-9bd0-4c59950d210f", + "value": "961468ddd3d0fa25beb8210c81ba620f9170ed30" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189002", + "object_id": "1576", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937910", + "to_ids": false, + "type": "text", + "uuid": "5a3cdbf6-acd8-4a36-a028-4c59950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Attribute": { + "category": "Network activity", + "distribution": "5", + "sharing_group_id": "0", + "to_ids": true, + "type": "domain", + "uuid": "5a3c3045-e354-4978-a6b4-49ad02de0b81", + "value": "faststoragefiles.org" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "164", + "object_id": "1576", + "object_uuid": "5a3cdbf6-f814-491f-9f93-4c59950d210f", + "referenced_id": "1188761", + "referenced_type": "0", + "referenced_uuid": "5a3c3045-e354-4978-a6b4-49ad02de0b81", + "relationship_type": "communicates-with", + "timestamp": "1513938210", + "uuid": "5a3cdd22-b7d8-4754-a108-4742950d210f" + } + ], + "comment": "Win32/Sednit.BO", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1576", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513938210", + "uuid": "5a3cdbf6-f814-491f-9f93-4c59950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189003", + "object_id": "1577", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937929", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cdc09-b428-4c0b-9969-c5ed950d210f", + "value": "a0719b50265505c8432616c0a4e14ed206981e95" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189004", + "object_id": "1577", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937929", + "to_ids": false, + "type": "text", + "uuid": "5a3cdc09-05d8-4356-ba52-c5ed950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Attribute": { + "category": "Network activity", + "distribution": "5", + "sharing_group_id": "0", + "to_ids": true, + "type": "domain", + "uuid": "5a3c3045-968c-4572-9f64-491502de0b81", + "value": "nethostnet.com" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "162", + "object_id": "1577", + "object_uuid": "5a3cdc09-6fbc-4ca1-bfaa-c5ed950d210f", + "referenced_id": "1188762", + "referenced_type": "0", + "referenced_uuid": "5a3c3045-968c-4572-9f64-491502de0b81", + "relationship_type": "communicates-with", + "timestamp": "1513938169", + "uuid": "5a3cdcf9-d5a4-4c8e-a201-45b1950d210f" + } + ], + "comment": "Win32/Sednit.BO", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1577", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513938169", + "uuid": "5a3cdc09-6fbc-4ca1-bfaa-c5ed950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189005", + "object_id": "1578", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937953", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cdc21-a170-4637-b139-4812950d210f", + "value": "2cf6436b99d11d9d1e0c488af518e35162ecbc9c" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189006", + "object_id": "1578", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937953", + "to_ids": false, + "type": "text", + "uuid": "5a3cdc21-3274-4800-9e91-41e2950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Attribute": { + "category": "Network activity", + "distribution": "5", + "sharing_group_id": "0", + "to_ids": true, + "type": "domain", + "uuid": "5a3c3045-e354-4978-a6b4-49ad02de0b81", + "value": "faststoragefiles.org" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "165", + "object_id": "1578", + "object_uuid": "5a3cdc21-856c-48bd-a757-4f4b950d210f", + "referenced_id": "1188761", + "referenced_type": "0", + "referenced_uuid": "5a3c3045-e354-4978-a6b4-49ad02de0b81", + "relationship_type": "communicates-with", + "timestamp": "1513938226", + "uuid": "5a3cdd32-3044-4895-8f18-4d06950d210f" + } + ], + "comment": "Win64/Sednit.Y", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1578", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513938226", + "uuid": "5a3cdc21-856c-48bd-a757-4f4b950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189007", + "object_id": "1579", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937975", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cdc37-cee0-43d0-9e20-4db6950d210f", + "value": "fec29b4f4dccc59770c65c128dfe4564d7c13d33" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189008", + "object_id": "1579", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937976", + "to_ids": false, + "type": "text", + "uuid": "5a3cdc38-ac24-44be-a1ed-4935950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Attribute": { + "category": "Network activity", + "distribution": "5", + "sharing_group_id": "0", + "to_ids": true, + "type": "domain", + "uuid": "5a3c3045-eb44-433f-a13a-44b902de0b81", + "value": "fsportal.net" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "163", + "object_id": "1579", + "object_uuid": "5a3cdc37-89e8-4a2d-823a-4af8950d210f", + "referenced_id": "1188763", + "referenced_type": "0", + "referenced_uuid": "5a3c3045-eb44-433f-a13a-44b902de0b81", + "relationship_type": "communicates-with", + "timestamp": "1513938189", + "uuid": "5a3cdd0d-d990-42ba-830d-5156950d210f" + } + ], + "comment": "Win64/Sednit.Y", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1579", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513938190", + "uuid": "5a3cdc37-89e8-4a2d-823a-4af8950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189009", + "object_id": "1580", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513937992", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cdc48-c74c-4b6e-8202-5156950d210f", + "value": "57d7f3d31c491f8aef4665ca4dd905c3c8a98795" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189010", + "object_id": "1580", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513937992", + "to_ids": false, + "type": "text", + "uuid": "5a3cdc48-55dc-420e-9b5d-5156950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Attribute": { + "category": "Network activity", + "distribution": "5", + "sharing_group_id": "0", + "to_ids": true, + "type": "domain", + "uuid": "5a3c3045-6a88-479d-b799-4d3d02de0b81", + "value": "fastdataexchange.org" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "161", + "object_id": "1580", + "object_uuid": "5a3cdc48-b9a0-4775-a03f-5156950d210f", + "referenced_id": "1188764", + "referenced_type": "0", + "referenced_uuid": "5a3c3045-6a88-479d-b799-4d3d02de0b81", + "relationship_type": "communicates-with", + "timestamp": "1513938129", + "uuid": "5a3cdcd1-c6cc-43d8-a2f4-4681950d210f" + } + ], + "comment": "Win64/Sednit.Z", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1580", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513938129", + "uuid": "5a3cdc48-b9a0-4775-a03f-5156950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189011", + "object_id": "1581", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513938011", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cdc5b-54a8-4e60-bc67-4c5a950d210f", + "value": "a3bf5b5cf5a5ef438a198a6f61f7225c0a4a7138" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189012", + "object_id": "1581", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513938011", + "to_ids": false, + "type": "text", + "uuid": "5a3cdc5b-b390-4183-aec7-4c5a950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Attribute": { + "category": "Network activity", + "distribution": "5", + "sharing_group_id": "0", + "to_ids": true, + "type": "domain", + "uuid": "5a3c3045-7480-4831-a5c4-48c802de0b81", + "value": "newfilmts.com" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "168", + "object_id": "1581", + "object_uuid": "5a3cdc5a-8760-4efa-949a-4c5a950d210f", + "referenced_id": "1188765", + "referenced_type": "0", + "referenced_uuid": "5a3c3045-7480-4831-a5c4-48c802de0b81", + "relationship_type": "communicates-with", + "timestamp": "1513938280", + "uuid": "5a3cdd68-7968-40d1-a0a9-5156950d210f" + } + ], + "comment": "Win32/Sednit.BO", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1581", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513938280", + "uuid": "5a3cdc5a-8760-4efa-949a-4c5a950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189013", + "object_id": "1582", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513938034", + "to_ids": true, + "type": "sha1", + "uuid": "5a3cdc72-ba30-4ecd-9d21-4654950d210f", + "value": "1958e722afd0dba266576922abc98aa505cf5f9a" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189014", + "object_id": "1582", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513938034", + "to_ids": false, + "type": "text", + "uuid": "5a3cdc72-0804-42c4-acfa-4ac5950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Attribute": { + "category": "Network activity", + "distribution": "5", + "sharing_group_id": "0", + "to_ids": true, + "type": "domain", + "uuid": "5a3c3045-7480-4831-a5c4-48c802de0b81", + "value": "newfilmts.com" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "167", + "object_id": "1582", + "object_uuid": "5a3cdc72-1538-4c66-af46-427b950d210f", + "referenced_id": "1188765", + "referenced_type": "0", + "referenced_uuid": "5a3c3045-7480-4831-a5c4-48c802de0b81", + "relationship_type": "communicates-with", + "timestamp": "1513938264", + "uuid": "5a3cdd58-9800-4bae-837c-4f20950d210f" + } + ], + "comment": "Win32/Sednit.BO", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1582", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513938264", + "uuid": "5a3cdc72-1538-4c66-af46-427b950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189015", + "object_id": "1583", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513939882", + "to_ids": true, + "type": "sha1", + "uuid": "5a3ce3aa-e104-481e-a7f4-4bc1950d210f", + "value": "9f6bed7d7f4728490117cbc85819c2e6c494251b" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189016", + "object_id": "1583", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513939882", + "to_ids": false, + "type": "text", + "uuid": "5a3ce3aa-74fc-48c7-af40-4c6a950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Object": { + "distribution": "5", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "uuid": "5a3ce58a-3198-4cb8-9d51-44e5950d210f" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "173", + "object_id": "1583", + "object_uuid": "5a3ce3a9-f070-4403-a1f6-4b8c950d210f", + "referenced_id": "1592", + "referenced_type": "1", + "referenced_uuid": "5a3ce58a-3198-4cb8-9d51-44e5950d210f", + "relationship_type": "communicates-with", + "timestamp": "1513947459", + "uuid": "5a3d0143-c300-4118-8afe-4a2d950d210f" + } + ], + "comment": "Win32/Sednit.AX\t", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1583", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513948642", + "uuid": "5a3ce3a9-f070-4403-a1f6-4b8c950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189017", + "object_id": "1584", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513939907", + "to_ids": true, + "type": "sha1", + "uuid": "5a3ce3c3-6d9c-48f4-93db-4a61950d210f", + "value": "4bc722a9b0492a50bd86a1341f02c74c0d773db7" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189018", + "object_id": "1584", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513939907", + "to_ids": false, + "type": "text", + "uuid": "5a3ce3c3-c38c-4e30-a904-4c8f950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Object": { + "distribution": "5", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "uuid": "5a3ce6ae-98d8-4270-b88f-47f2950d210f" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "188", + "object_id": "1584", + "object_uuid": "5a3ce3c3-34b4-4e1f-b238-4399950d210f", + "referenced_id": "1603", + "referenced_type": "1", + "referenced_uuid": "5a3ce6ae-98d8-4270-b88f-47f2950d210f", + "relationship_type": "communicates-with", + "timestamp": "1513948518", + "uuid": "5a3d0566-34fc-4a62-b2a5-4f91950d210f" + } + ], + "comment": "Win32/Sednit.BS", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1584", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513948535", + "uuid": "5a3ce3c3-34b4-4e1f-b238-4399950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189019", + "object_id": "1585", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513939924", + "to_ids": true, + "type": "sha1", + "uuid": "5a3ce3d4-9168-4e23-8b64-485a950d210f", + "value": "ab354807e687993fbeb1b325eb6e4ab38d428a1e" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189020", + "object_id": "1585", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513939924", + "to_ids": false, + "type": "text", + "uuid": "5a3ce3d4-27e0-4366-943f-4b9a950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Object": { + "distribution": "5", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "uuid": "5a3ce6a1-3f1c-4d5d-bac7-406d950d210f" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "189", + "object_id": "1585", + "object_uuid": "5a3ce3d4-07bc-4af3-90fc-4798950d210f", + "referenced_id": "1602", + "referenced_type": "1", + "referenced_uuid": "5a3ce6a1-3f1c-4d5d-bac7-406d950d210f", + "relationship_type": "communicates-with", + "timestamp": "1513948528", + "uuid": "5a3d0570-a86c-4264-a43a-4125950d210f" + } + ], + "comment": "Win32/Sednit.BS", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1585", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513948597", + "uuid": "5a3ce3d4-07bc-4af3-90fc-4798950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189021", + "object_id": "1586", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513939946", + "to_ids": true, + "type": "sha1", + "uuid": "5a3ce3ea-8dbc-4cf4-997f-448b950d210f", + "value": "9c47ca3883196b3a84d67676a804ff50e22b0a9f" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189022", + "object_id": "1586", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513939946", + "to_ids": false, + "type": "text", + "uuid": "5a3ce3ea-e714-444e-ad9b-40b0950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Object": { + "distribution": "5", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "uuid": "5a3ce68d-1940-4ea6-becd-44fe950d210f" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "190", + "object_id": "1586", + "object_uuid": "5a3ce3ea-580c-477c-9b73-4e57950d210f", + "referenced_id": "1601", + "referenced_type": "1", + "referenced_uuid": "5a3ce68d-1940-4ea6-becd-44fe950d210f", + "relationship_type": "communicates-with", + "timestamp": "1513948614", + "uuid": "5a3d05c6-0618-4520-9549-48a0950d210f" + } + ], + "comment": "Win32/Sednit.BR", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1586", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513948626", + "uuid": "5a3ce3ea-580c-477c-9b73-4e57950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189023", + "object_id": "1587", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513939972", + "to_ids": true, + "type": "sha1", + "uuid": "5a3ce404-7bfc-4316-bd32-55ea950d210f", + "value": "8a68f26d01372114f660e32ac4c9117e5d0577f1" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189024", + "object_id": "1587", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513939972", + "to_ids": false, + "type": "text", + "uuid": "5a3ce404-7224-4525-922a-55ea950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Object": { + "distribution": "5", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "uuid": "5a3ce680-90d4-478d-95db-48a6950d210f" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "182", + "object_id": "1587", + "object_uuid": "5a3ce404-efc0-4f15-864e-55ea950d210f", + "referenced_id": "1600", + "referenced_type": "1", + "referenced_uuid": "5a3ce680-90d4-478d-95db-48a6950d210f", + "relationship_type": "communicates-with", + "timestamp": "1513948044", + "uuid": "5a3d038c-1cc8-4d9c-87ab-c5ed950d210f" + } + ], + "comment": "Win32/Sednit.BN", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1587", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513948073", + "uuid": "5a3ce404-efc0-4f15-864e-55ea950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189025", + "object_id": "1588", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513939991", + "to_ids": true, + "type": "sha1", + "uuid": "5a3ce417-62a4-4d46-9a87-55ea950d210f", + "value": "476fc1d31722ac26b46154cbf0c631d60268b28a" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189026", + "object_id": "1588", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513939991", + "to_ids": false, + "type": "text", + "uuid": "5a3ce417-43f0-494d-ac2e-55ea950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Object": { + "distribution": "5", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "uuid": "5a3ce66e-70b4-47e7-b965-46f6950d210f" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "187", + "object_id": "1588", + "object_uuid": "5a3ce417-7cd4-4c36-8a73-55ea950d210f", + "referenced_id": "1599", + "referenced_type": "1", + "referenced_uuid": "5a3ce66e-70b4-47e7-b965-46f6950d210f", + "relationship_type": "communicates-with", + "timestamp": "1513948483", + "uuid": "5a3d0543-8f74-4086-aafc-418a950d210f" + } + ], + "comment": "Win32/Sednit.BN", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1588", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513948498", + "uuid": "5a3ce417-7cd4-4c36-8a73-55ea950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189027", + "object_id": "1589", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513940012", + "to_ids": true, + "type": "sha1", + "uuid": "5a3ce42c-836c-49e7-a9f3-4a5f950d210f", + "value": "f9fd3f1d8da4ffd6a494228b934549d09e3c59d1" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189028", + "object_id": "1589", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513940012", + "to_ids": false, + "type": "text", + "uuid": "5a3ce42c-4c88-4940-94b8-4084950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Object": { + "distribution": "5", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "uuid": "5a3ce60a-6db8-4212-b194-4339950d210f" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "183", + "object_id": "1589", + "object_uuid": "5a3ce42b-2e0c-4a26-b6c8-47a3950d210f", + "referenced_id": "1594", + "referenced_type": "1", + "referenced_uuid": "5a3ce60a-6db8-4212-b194-4339950d210f", + "relationship_type": "communicates-with", + "timestamp": "1513948106", + "uuid": "5a3d03ca-2398-4060-b13c-404a950d210f" + }, + { + "Object": { + "distribution": "5", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "uuid": "5a3ce61a-c1f0-4c7c-b815-4fa9950d210f" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "184", + "object_id": "1589", + "object_uuid": "5a3ce42b-2e0c-4a26-b6c8-47a3950d210f", + "referenced_id": "1595", + "referenced_type": "1", + "referenced_uuid": "5a3ce61a-c1f0-4c7c-b815-4fa9950d210f", + "relationship_type": "communicates-with", + "timestamp": "1513948117", + "uuid": "5a3d03d5-6d8c-4dfb-b193-4002950d210f" + } + ], + "comment": "Win32/Sednit.BN", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1589", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513948128", + "uuid": "5a3ce42b-2e0c-4a26-b6c8-47a3950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189029", + "object_id": "1590", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513940027", + "to_ids": true, + "type": "sha1", + "uuid": "5a3ce43b-6738-4a14-a318-4d65950d210f", + "value": "e338d49c270baf64363879e5eecb8fa6bdde8ad9" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189030", + "object_id": "1590", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513940027", + "to_ids": false, + "type": "text", + "uuid": "5a3ce43b-3a10-4d78-9ee2-485c950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Object": { + "distribution": "5", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "uuid": "5a3ce5f8-3418-4f7b-ae41-4bca950d210f" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "186", + "object_id": "1590", + "object_uuid": "5a3ce43a-5478-4f65-95b2-4e1e950d210f", + "referenced_id": "1593", + "referenced_type": "1", + "referenced_uuid": "5a3ce5f8-3418-4f7b-ae41-4bca950d210f", + "relationship_type": "communicates-with", + "timestamp": "1513948320", + "uuid": "5a3d04a0-9d28-47c3-a12c-465b950d210f" + } + ], + "comment": "Win32/Sednit.BG", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1590", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513948339", + "uuid": "5a3ce43a-5478-4f65-95b2-4e1e950d210f" + }, + { + "Attribute": [ + { + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189031", + "object_id": "1591", + "object_relation": "sha1", + "sharing_group_id": "0", + "timestamp": "1513940042", + "to_ids": true, + "type": "sha1", + "uuid": "5a3ce44a-2ea4-4526-8bbc-c328950d210f", + "value": "6e167da3c5d887fa2e58da848a2245d11b6c5ad6" + }, + { + "category": "Other", + "comment": "", + "deleted": false, + "disable_correlation": true, + "distribution": "5", + "event_id": "9747", + "id": "1189032", + "object_id": "1591", + "object_relation": "state", + "sharing_group_id": "0", + "timestamp": "1513940042", + "to_ids": false, + "type": "text", + "uuid": "5a3ce44a-5118-4142-97f0-c328950d210f", + "value": "Malicious" + } + ], + "ObjectReference": [ + { + "Object": { + "distribution": "5", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "uuid": "5a3ce64e-8bf8-4dc6-be49-437f950d210f" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "170", + "object_id": "1591", + "object_uuid": "5a3ce44a-ce70-42b7-80b8-c328950d210f", + "referenced_id": "1597", + "referenced_type": "1", + "referenced_uuid": "5a3ce64e-8bf8-4dc6-be49-437f950d210f", + "relationship_type": "communicates-with", + "timestamp": "1513940734", + "uuid": "5a3ce6fe-b0c4-44df-a609-419a950d210f" + }, + { + "Object": { + "distribution": "5", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "uuid": "5a3ce65c-fc40-4585-817e-4ca3950d210f" + }, + "comment": "", + "deleted": false, + "event_id": "9747", + "id": "171", + "object_id": "1591", + "object_uuid": "5a3ce44a-ce70-42b7-80b8-c328950d210f", + "referenced_id": "1598", + "referenced_type": "1", + "referenced_uuid": "5a3ce65c-fc40-4585-817e-4ca3950d210f", + "relationship_type": "communicates-with", + "timestamp": "1513940753", + "uuid": "5a3ce711-a0dc-4dbe-b59e-495a950d210f" + } + ], + "comment": "Win32/Sednit.BG", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "9747", + "id": "1591", + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "8", + "timestamp": "1513940753", + "uuid": "5a3ce44a-ce70-42b7-80b8-c328950d210f" + }, + { + "Attribute": [ + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189033", + "object_id": "1592", + "object_relation": "ip", + "sharing_group_id": "0", + "timestamp": "1513940362", + "to_ids": true, + "type": "ip-dst", + "uuid": "5a3ce58a-fcd8-48d5-8b4a-4fd9950d210f", + "value": "87.236.211.182" + }, + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189034", + "object_id": "1592", + "object_relation": "domain", + "sharing_group_id": "0", + "timestamp": "1513940362", + "to_ids": true, + "type": "domain", + "uuid": "5a3ce58a-6e14-48ea-9746-48f2950d210f", + "value": "servicecdp.com" + } + ], + "comment": "", + "deleted": false, + "description": "A domain and IP address seen as a tuple in a specific time frame.", + "distribution": "5", + "event_id": "9747", + "id": "1592", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "template_uuid": "43b3b146-77eb-4931-b4cc-b66c60f28734", + "template_version": "5", + "timestamp": "1513940362", + "uuid": "5a3ce58a-3198-4cb8-9d51-44e5950d210f" + }, + { + "Attribute": [ + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189035", + "object_id": "1593", + "object_relation": "ip", + "sharing_group_id": "0", + "timestamp": "1513940472", + "to_ids": true, + "type": "ip-dst", + "uuid": "5a3ce5f8-99b4-41a2-915a-4bf8950d210f", + "value": "95.215.45.43" + }, + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189036", + "object_id": "1593", + "object_relation": "domain", + "sharing_group_id": "0", + "timestamp": "1513940472", + "to_ids": true, + "type": "domain", + "uuid": "5a3ce5f8-62c8-4f04-89c2-4aeb950d210f", + "value": "wmdmediacodecs.com" + } + ], + "comment": "", + "deleted": false, + "description": "A domain and IP address seen as a tuple in a specific time frame.", + "distribution": "5", + "event_id": "9747", + "id": "1593", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "template_uuid": "43b3b146-77eb-4931-b4cc-b66c60f28734", + "template_version": "5", + "timestamp": "1513940472", + "uuid": "5a3ce5f8-3418-4f7b-ae41-4bca950d210f" + }, + { + "Attribute": [ + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189037", + "object_id": "1594", + "object_relation": "ip", + "sharing_group_id": "0", + "timestamp": "1513940490", + "to_ids": true, + "type": "ip-dst", + "uuid": "5a3ce60a-cc50-4553-bfff-4ea9950d210f", + "value": "89.45.67.144" + }, + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189038", + "object_id": "1594", + "object_relation": "domain", + "sharing_group_id": "0", + "timestamp": "1513940491", + "to_ids": true, + "type": "domain", + "uuid": "5a3ce60b-e648-4667-8432-4ba8950d210f", + "value": "mvband.net" + } + ], + "comment": "", + "deleted": false, + "description": "A domain and IP address seen as a tuple in a specific time frame.", + "distribution": "5", + "event_id": "9747", + "id": "1594", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "template_uuid": "43b3b146-77eb-4931-b4cc-b66c60f28734", + "template_version": "5", + "timestamp": "1513940490", + "uuid": "5a3ce60a-6db8-4212-b194-4339950d210f" + }, + { + "Attribute": [ + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189039", + "object_id": "1595", + "object_relation": "ip", + "sharing_group_id": "0", + "timestamp": "1513940506", + "to_ids": true, + "type": "ip-dst", + "uuid": "5a3ce61a-4458-4c36-866e-44e9950d210f", + "value": "89.33.246.117" + }, + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189040", + "object_id": "1595", + "object_relation": "domain", + "sharing_group_id": "0", + "timestamp": "1513940506", + "to_ids": true, + "type": "domain", + "uuid": "5a3ce61a-f820-4a43-b3d9-47e5950d210f", + "value": "mvtband.net" + } + ], + "comment": "", + "deleted": false, + "description": "A domain and IP address seen as a tuple in a specific time frame.", + "distribution": "5", + "event_id": "9747", + "id": "1595", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "template_uuid": "43b3b146-77eb-4931-b4cc-b66c60f28734", + "template_version": "5", + "timestamp": "1513940506", + "uuid": "5a3ce61a-c1f0-4c7c-b815-4fa9950d210f" + }, + { + "Attribute": [ + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189041", + "object_id": "1596", + "object_relation": "ip", + "sharing_group_id": "0", + "timestamp": "1513940542", + "to_ids": true, + "type": "ip-dst", + "uuid": "5a3ce63e-66d4-483f-bae6-44f6950d210f", + "value": "87.236.211.182" + }, + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189042", + "object_id": "1596", + "object_relation": "domain", + "sharing_group_id": "0", + "timestamp": "1513940542", + "to_ids": true, + "type": "domain", + "uuid": "5a3ce63e-0d88-405b-82a9-43b5950d210f", + "value": "servicecdp.com" + } + ], + "comment": "", + "deleted": false, + "description": "A domain and IP address seen as a tuple in a specific time frame.", + "distribution": "5", + "event_id": "9747", + "id": "1596", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "template_uuid": "43b3b146-77eb-4931-b4cc-b66c60f28734", + "template_version": "5", + "timestamp": "1513940542", + "uuid": "5a3ce63e-0240-46f5-b9ed-4759950d210f" + }, + { + "Attribute": [ + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189043", + "object_id": "1597", + "object_relation": "ip", + "sharing_group_id": "0", + "timestamp": "1513940558", + "to_ids": true, + "type": "ip-dst", + "uuid": "5a3ce64e-d7a8-4817-a132-4c72950d210f", + "value": "185.156.173.70" + }, + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189044", + "object_id": "1597", + "object_relation": "domain", + "sharing_group_id": "0", + "timestamp": "1513940558", + "to_ids": true, + "type": "domain", + "uuid": "5a3ce64e-243c-4931-b733-403c950d210f", + "value": "runvercheck.com" + } + ], + "comment": "", + "deleted": false, + "description": "A domain and IP address seen as a tuple in a specific time frame.", + "distribution": "5", + "event_id": "9747", + "id": "1597", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "template_uuid": "43b3b146-77eb-4931-b4cc-b66c60f28734", + "template_version": "5", + "timestamp": "1513940558", + "uuid": "5a3ce64e-8bf8-4dc6-be49-437f950d210f" + }, + { + "Attribute": [ + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189045", + "object_id": "1598", + "object_relation": "ip", + "sharing_group_id": "0", + "timestamp": "1513940572", + "to_ids": true, + "type": "ip-dst", + "uuid": "5a3ce65c-bf78-4b78-bafd-4cf6950d210f", + "value": "191.101.31.96" + }, + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189046", + "object_id": "1598", + "object_relation": "domain", + "sharing_group_id": "0", + "timestamp": "1513940572", + "to_ids": true, + "type": "domain", + "uuid": "5a3ce65c-8140-4146-a927-45e4950d210f", + "value": "remsupport.org" + } + ], + "comment": "", + "deleted": false, + "description": "A domain and IP address seen as a tuple in a specific time frame.", + "distribution": "5", + "event_id": "9747", + "id": "1598", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "template_uuid": "43b3b146-77eb-4931-b4cc-b66c60f28734", + "template_version": "5", + "timestamp": "1513940572", + "uuid": "5a3ce65c-fc40-4585-817e-4ca3950d210f" + }, + { + "Attribute": [ + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189047", + "object_id": "1599", + "object_relation": "ip", + "sharing_group_id": "0", + "timestamp": "1513940591", + "to_ids": true, + "type": "ip-dst", + "uuid": "5a3ce66f-150c-43ec-a3ff-4aa5950d210f", + "value": "89.187.150.44" + }, + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189048", + "object_id": "1599", + "object_relation": "domain", + "sharing_group_id": "0", + "timestamp": "1513940591", + "to_ids": true, + "type": "domain", + "uuid": "5a3ce66f-466c-478e-8064-4b42950d210f", + "value": "viters.org" + } + ], + "comment": "", + "deleted": false, + "description": "A domain and IP address seen as a tuple in a specific time frame.", + "distribution": "5", + "event_id": "9747", + "id": "1599", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "template_uuid": "43b3b146-77eb-4931-b4cc-b66c60f28734", + "template_version": "5", + "timestamp": "1513940590", + "uuid": "5a3ce66e-70b4-47e7-b965-46f6950d210f" + }, + { + "Attribute": [ + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189049", + "object_id": "1600", + "object_relation": "ip", + "sharing_group_id": "0", + "timestamp": "1513940608", + "to_ids": true, + "type": "ip-dst", + "uuid": "5a3ce680-7b04-466d-b187-4301950d210f", + "value": "146.185.253.132" + }, + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189050", + "object_id": "1600", + "object_relation": "domain", + "sharing_group_id": "0", + "timestamp": "1513940608", + "to_ids": true, + "type": "domain", + "uuid": "5a3ce680-12f4-4001-9f86-4aa4950d210f", + "value": "myinvestgroup.com" + } + ], + "comment": "", + "deleted": false, + "description": "A domain and IP address seen as a tuple in a specific time frame.", + "distribution": "5", + "event_id": "9747", + "id": "1600", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "template_uuid": "43b3b146-77eb-4931-b4cc-b66c60f28734", + "template_version": "5", + "timestamp": "1513940608", + "uuid": "5a3ce680-90d4-478d-95db-48a6950d210f" + }, + { + "Attribute": [ + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189051", + "object_id": "1601", + "object_relation": "ip", + "sharing_group_id": "0", + "timestamp": "1513940621", + "to_ids": true, + "type": "ip-dst", + "uuid": "5a3ce68d-0108-4557-8921-4377950d210f", + "value": "86.106.131.141" + }, + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189052", + "object_id": "1601", + "object_relation": "domain", + "sharing_group_id": "0", + "timestamp": "1513940622", + "to_ids": true, + "type": "domain", + "uuid": "5a3ce68e-54d0-4c67-8c4c-4dea950d210f", + "value": "space-delivery.com" + } + ], + "comment": "", + "deleted": false, + "description": "A domain and IP address seen as a tuple in a specific time frame.", + "distribution": "5", + "event_id": "9747", + "id": "1601", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "template_uuid": "43b3b146-77eb-4931-b4cc-b66c60f28734", + "template_version": "5", + "timestamp": "1513940621", + "uuid": "5a3ce68d-1940-4ea6-becd-44fe950d210f" + }, + { + "Attribute": [ + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189054", + "object_id": "1602", + "object_relation": "ip", + "sharing_group_id": "0", + "timestamp": "1513940642", + "to_ids": true, + "type": "ip-dst", + "uuid": "5a3ce6a2-4a38-4b90-8d74-4f10950d210f", + "value": "89.34.111.160" + }, + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189055", + "object_id": "1602", + "object_relation": "domain", + "sharing_group_id": "0", + "timestamp": "1513940642", + "to_ids": true, + "type": "domain", + "uuid": "5a3ce6a2-ffa4-4afb-89ab-42a6950d210f", + "value": "satellitedeluxpanorama.com" + } + ], + "comment": "", + "deleted": false, + "description": "A domain and IP address seen as a tuple in a specific time frame.", + "distribution": "5", + "event_id": "9747", + "id": "1602", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "template_uuid": "43b3b146-77eb-4931-b4cc-b66c60f28734", + "template_version": "5", + "timestamp": "1513940641", + "uuid": "5a3ce6a1-3f1c-4d5d-bac7-406d950d210f" + }, + { + "Attribute": [ + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189056", + "object_id": "1603", + "object_relation": "ip", + "sharing_group_id": "0", + "timestamp": "1513940654", + "to_ids": true, + "type": "ip-dst", + "uuid": "5a3ce6ae-601c-44b8-8eec-4a5f950d210f", + "value": "185.216.35.26" + }, + { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "9747", + "id": "1189057", + "object_id": "1603", + "object_relation": "domain", + "sharing_group_id": "0", + "timestamp": "1513940654", + "to_ids": true, + "type": "domain", + "uuid": "5a3ce6ae-3b00-420a-82fd-45fb950d210f", + "value": "webviewres.net" + } + ], + "comment": "", + "deleted": false, + "description": "A domain and IP address seen as a tuple in a specific time frame.", + "distribution": "5", + "event_id": "9747", + "id": "1603", + "meta-category": "network", + "name": "domain-ip", + "sharing_group_id": "0", + "template_uuid": "43b3b146-77eb-4931-b4cc-b66c60f28734", + "template_version": "5", + "timestamp": "1513940654", + "uuid": "5a3ce6ae-98d8-4270-b88f-47f2950d210f" + } + ], + "Org": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "Orgc": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "RelatedEvent": [ + { + "Event": { + "Org": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "Orgc": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "analysis": "2", + "date": "2017-12-14", + "distribution": "3", + "id": "9616", + "info": "OSINT - Attackers Deploy New ICS Attack Framework “TRITON” and Cause Operational Disruption to Critical Infrastructure", + "org_id": "2", + "orgc_id": "2", + "published": false, + "threat_level_id": "3", + "timestamp": "1513674510", + "uuid": "5a329d19-03e0-4eaa-8b4d-4310950d210f" + } + }, + { + "Event": { + "Org": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "Orgc": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "analysis": "2", + "date": "2017-12-07", + "distribution": "3", + "id": "9552", + "info": "OSINT - Master Channel: The Boleto Mestre Campaign Targets Brazil", + "org_id": "2", + "orgc_id": "2", + "published": false, + "threat_level_id": "3", + "timestamp": "1512657975", + "uuid": "5a2943a3-c574-44bb-8e68-45de950d210f" + } + }, + { + "Event": { + "Org": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "Orgc": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "analysis": "0", + "date": "2017-11-27", + "distribution": "3", + "id": "9513", + "info": "OSINT - Tizi: Detecting and blocking socially engineered spyware on Android", + "org_id": "2", + "orgc_id": "2", + "published": true, + "threat_level_id": "3", + "timestamp": "1512356440", + "uuid": "5a23a972-e6a0-4a05-b505-4e8f02de0b81" + } + }, + { + "Event": { + "Org": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "Orgc": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "analysis": "2", + "date": "2017-11-07", + "distribution": "3", + "id": "9309", + "info": "OSINT - Threat Group APT28 Slips Office Malware into Doc Citing NYC Terror Attack", + "org_id": "2", + "orgc_id": "2", + "published": true, + "threat_level_id": "3", + "timestamp": "1511385862", + "uuid": "5a021bc2-8e0c-4ac5-b048-cc3e02de0b81" + } + }, + { + "Event": { + "Org": { + "id": "291", + "name": "NCSC-NL", + "uuid": "5697b0c4-9474-4336-b675-28140a950b0b" + }, + "Orgc": { + "id": "291", + "name": "NCSC-NL", + "uuid": "5697b0c4-9474-4336-b675-28140a950b0b" + }, + "analysis": "2", + "date": "2017-10-23", + "distribution": "3", + "id": "9208", + "info": "Talos: “Cyber Conflict” Decoy Document Used In Real Cyber Conflict", + "org_id": "291", + "orgc_id": "291", + "published": true, + "threat_level_id": "2", + "timestamp": "1510088616", + "uuid": "59ed9c81-6484-47a9-aab4-191d0a950b0c" + } + }, + { + "Event": { + "Org": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "Orgc": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "analysis": "2", + "date": "2017-08-11", + "distribution": "3", + "id": "8798", + "info": "OSINT - APT28 Targets Hospitality Sector, Presents Threat to Travelers", + "org_id": "2", + "orgc_id": "2", + "published": true, + "threat_level_id": "3", + "timestamp": "1502460096", + "uuid": "598db7fd-47a8-45f8-9414-408b02de0b81" + } + }, + { + "Event": { + "Org": { + "id": "231", + "name": "kingfisherops.com", + "uuid": "566ff5f4-7020-4089-9003-4374950d210f" + }, + "Orgc": { + "id": "204", + "name": "CERT-BUND", + "uuid": "56a64d7a-63dc-4471-bce9-4accc25ed029" + }, + "analysis": "0", + "date": "2017-07-25", + "distribution": "3", + "id": "8750", + "info": "European Defence Agency lure drops mssuppa.dat", + "org_id": "231", + "orgc_id": "204", + "published": true, + "threat_level_id": "2", + "timestamp": "1500967989", + "uuid": "5976f294-a844-44fe-a4f0-6c67c25ed029" + } + }, + { + "Event": { + "Org": { + "id": "277", + "name": "inthreat.com", + "uuid": "5697b91d-2090-441f-b153-75e895ca48b7" + }, + "Orgc": { + "id": "277", + "name": "inthreat.com", + "uuid": "5697b91d-2090-441f-b153-75e895ca48b7" + }, + "analysis": "2", + "date": "2017-05-11", + "distribution": "3", + "id": "7820", + "info": "APT28-Sednit adds two zero-day exploits using ‘Trump’s attack on Syria’ as a decoy", + "org_id": "277", + "orgc_id": "277", + "published": true, + "threat_level_id": "2", + "timestamp": "1494824291", + "uuid": "59147a22-3100-4779-9377-360395ca48b7" + } + }, + { + "Event": { + "Org": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "Orgc": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "analysis": "2", + "date": "2017-05-09", + "distribution": "3", + "id": "7801", + "info": "OSINT - EPS Processing Zero-Days Exploited by Multiple Threat Actors", + "org_id": "2", + "orgc_id": "2", + "published": true, + "threat_level_id": "3", + "timestamp": "1494354378", + "uuid": "59120865-27e0-4e6d-9b74-4a9f950d210f" + } + }, + { + "Event": { + "Org": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "Orgc": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "analysis": "0", + "date": "2016-12-29", + "distribution": "3", + "id": "5667", + "info": "OSINT - GRIZZLY STEPPE – Russian Malicious Cyber Activity", + "org_id": "2", + "orgc_id": "2", + "published": true, + "threat_level_id": "3", + "timestamp": "1494853878", + "uuid": "58658c15-54ac-43c3-9beb-414502de0b81" + } + }, + { + "Event": { + "Org": { + "id": "277", + "name": "inthreat.com", + "uuid": "5697b91d-2090-441f-b153-75e895ca48b7" + }, + "Orgc": { + "id": "277", + "name": "inthreat.com", + "uuid": "5697b91d-2090-441f-b153-75e895ca48b7" + }, + "analysis": "2", + "date": "2016-12-20", + "distribution": "1", + "id": "5616", + "info": "APT28-The Sofacy Group's DealersChoice Attacks Continue", + "org_id": "277", + "orgc_id": "277", + "published": true, + "threat_level_id": "2", + "timestamp": "1494829249", + "uuid": "58594faf-e98c-4c03-a58c-43cf95ca48b7" + } + }, + { + "Event": { + "Org": { + "id": "291", + "name": "NCSC-NL", + "uuid": "5697b0c4-9474-4336-b675-28140a950b0b" + }, + "Orgc": { + "id": "291", + "name": "NCSC-NL", + "uuid": "5697b0c4-9474-4336-b675-28140a950b0b" + }, + "analysis": "1", + "date": "2016-11-09", + "distribution": "3", + "id": "5348", + "info": "[APT-28/Sofacy]Pawn Storm Ramps Up [European Government] Spear-phishing Before Zero-Days Get Patched", + "org_id": "291", + "orgc_id": "291", + "published": true, + "threat_level_id": "1", + "timestamp": "1481709638", + "uuid": "582341ff-0830-4b32-aaba-08640a950b0c" + } + }, + { + "Event": { + "Org": { + "id": "74", + "name": "PwC.lu", + "uuid": "55f6ea61-4f74-40b6-a6df-4ff9950d210f" + }, + "Orgc": { + "id": "325", + "name": "CUDESO", + "uuid": "56c42374-fdb8-4544-a218-41ffc0a8ab16" + }, + "analysis": "2", + "date": "2016-11-09", + "distribution": "3", + "id": "5641", + "info": "Pawn Storm Ramps Up Spear-phishing Before Zero-Days Get Patched", + "org_id": "74", + "orgc_id": "325", + "published": true, + "threat_level_id": "2", + "timestamp": "1478712711", + "uuid": "58235d0e-34d4-41c1-9a2e-04dcc0a8ab16" + } + }, + { + "Event": { + "Org": { + "id": "335", + "name": "Orange CERT-CC", + "uuid": "5707ccb5-e330-4e25-a193-41d4950d210f" + }, + "Orgc": { + "id": "335", + "name": "Orange CERT-CC", + "uuid": "5707ccb5-e330-4e25-a193-41d4950d210f" + }, + "analysis": "0", + "date": "2016-10-18", + "distribution": "0", + "id": "5163", + "info": "Orange-CERT-CC Test #01", + "org_id": "335", + "orgc_id": "335", + "published": false, + "threat_level_id": "3", + "timestamp": "1476782422", + "uuid": "5805e8a5-611c-498b-839b-bd57950d210f" + } + }, + { + "Event": { + "Org": { + "id": "278", + "name": "TDC.dk", + "uuid": "56a5d575-2ff4-4738-a2ee-59be950d210f" + }, + "Orgc": { + "id": "278", + "name": "TDC.dk", + "uuid": "56a5d575-2ff4-4738-a2ee-59be950d210f" + }, + "analysis": "2", + "date": "2016-10-17", + "distribution": "3", + "id": "5165", + "info": "OSINT: ‘DealersChoice’ is Sofacy’s Flash Player Exploit Platform", + "org_id": "278", + "orgc_id": "278", + "published": true, + "threat_level_id": "1", + "timestamp": "1476789563", + "uuid": "580602f6-f8b8-4ac3-9813-7bf7bce2ab96" + } + }, + { + "Event": { + "Org": { + "id": "412", + "name": "TS", + "uuid": "57470e61-3384-491d-a56f-1bb75b86d7e5" + }, + "Orgc": { + "id": "412", + "name": "TS", + "uuid": "57470e61-3384-491d-a56f-1bb75b86d7e5" + }, + "analysis": "2", + "date": "2016-08-19", + "distribution": "1", + "id": "4710", + "info": "bullettin.doc sample, linked to APT28 campaign", + "org_id": "412", + "orgc_id": "412", + "published": true, + "threat_level_id": "1", + "timestamp": "1476776982", + "uuid": "57b7248f-283c-442e-8e02-2d0f5b86d7e5" + } + }, + { + "Event": { + "Org": { + "id": "277", + "name": "inthreat.com", + "uuid": "5697b91d-2090-441f-b153-75e895ca48b7" + }, + "Orgc": { + "id": "277", + "name": "inthreat.com", + "uuid": "5697b91d-2090-441f-b153-75e895ca48b7" + }, + "analysis": "2", + "date": "2016-06-20", + "distribution": "3", + "id": "4172", + "info": "APT28 and APT29 - Inside the DNC Breaches", + "org_id": "277", + "orgc_id": "277", + "published": true, + "threat_level_id": "2", + "timestamp": "1494829231", + "uuid": "5767c102-c170-4124-ae3d-7bef95ca48b7" + } + }, + { + "Event": { + "Org": { + "id": "347", + "name": "incibe.es", + "uuid": "5720623c-129c-4989-ae9d-4a11950d210f" + }, + "Orgc": { + "id": "665", + "name": "INCIBE", + "uuid": "56fa4fe4-f528-4480-8332-1ba3c0a80a8c" + }, + "analysis": "2", + "date": "2016-06-16", + "distribution": "3", + "id": "6131", + "info": "New Sofacy (APT28) attacks against a US Government Agency", + "org_id": "347", + "orgc_id": "665", + "published": true, + "threat_level_id": "1", + "timestamp": "1488792538", + "uuid": "5762a86a-e314-4e4e-ba5a-51c5c0a80a8e" + } + }, + { + "Event": { + "Org": { + "id": "26", + "name": "CthulhuSPRL.be", + "uuid": "55f6ea5f-fd34-43b8-ac1d-40cb950d210f" + }, + "Orgc": { + "id": "26", + "name": "CthulhuSPRL.be", + "uuid": "55f6ea5f-fd34-43b8-ac1d-40cb950d210f" + }, + "analysis": "2", + "date": "2016-06-15", + "distribution": "3", + "id": "3987", + "info": "OSINT New Sofacy Attacks Against US Government Agency by Palo Alto Unit 42", + "org_id": "26", + "orgc_id": "26", + "published": true, + "threat_level_id": "1", + "timestamp": "1466000907", + "uuid": "57613790-f6b4-4895-943f-4467950d210f" + } + }, + { + "Event": { + "Org": { + "id": "278", + "name": "TDC.dk", + "uuid": "56a5d575-2ff4-4738-a2ee-59be950d210f" + }, + "Orgc": { + "id": "325", + "name": "CUDESO", + "uuid": "56c42374-fdb8-4544-a218-41ffc0a8ab16" + }, + "analysis": "2", + "date": "2016-06-14", + "distribution": "3", + "id": "4183", + "info": "New Sofacy Attacks Against US Government Agency", + "org_id": "278", + "orgc_id": "325", + "published": true, + "threat_level_id": "2", + "timestamp": "1467289109", + "uuid": "57607369-2490-444a-9034-049fc0a8ab16" + } + } + ], + "Tag": [ + { + "colour": "#00d622", + "exportable": true, + "hide_tag": false, + "id": "2", + "name": "tlp:white", + "user_id": "0" + }, + { + "colour": "#ef0081", + "exportable": true, + "hide_tag": false, + "id": "2986", + "name": "workflow:state=\"incomplete\"", + "user_id": "0" + }, + { + "colour": "#810046", + "exportable": true, + "hide_tag": false, + "id": "2979", + "name": "workflow:todo=\"create-missing-misp-galaxy-cluster-values\"", + "user_id": "0" + }, + { + "colour": "#91004e", + "exportable": true, + "hide_tag": false, + "id": "2980", + "name": "workflow:todo=\"create-missing-misp-galaxy-cluster\"", + "user_id": "0" + }, + { + "colour": "#12e000", + "exportable": true, + "hide_tag": false, + "id": "1100", + "name": "misp-galaxy:threat-actor=\"Sofacy\"", + "user_id": "0" + }, + { + "colour": "#0088cc", + "exportable": true, + "hide_tag": false, + "id": "3007", + "name": "misp-galaxy:exploit-kit=\"Sednit EK\"", + "user_id": "0" + }, + { + "colour": "#0088cc", + "exportable": true, + "hide_tag": false, + "id": "2215", + "name": "misp-galaxy:tool=\"GAMEFISH\"", + "user_id": "0" + }, + { + "colour": "#0088cc", + "exportable": true, + "hide_tag": false, + "id": "3008", + "name": "misp-galaxy:mitre-malware=\"JHUHUGIT\"", + "user_id": "0" + }, + { + "colour": "#0c9900", + "exportable": true, + "hide_tag": false, + "id": "1012", + "name": "misp-galaxy:tool=\"X-Tunnel\"", + "user_id": "0" + }, + { + "colour": "#0088cc", + "exportable": true, + "hide_tag": false, + "id": "3009", + "name": "misp-galaxy:mitre-malware=\"XTunnel\"", + "user_id": "0" + }, + { + "colour": "#0088cc", + "exportable": true, + "hide_tag": false, + "id": "3010", + "name": "misp-galaxy:mitre-malware=\"ADVSTORESHELL\"", + "user_id": "0" + }, + { + "colour": "#0088cc", + "exportable": true, + "hide_tag": false, + "id": "3011", + "name": "misp-galaxy:tool=\"EVILTOSS\"", + "user_id": "0" + }, + { + "colour": "#0088cc", + "exportable": true, + "hide_tag": false, + "id": "3012", + "name": "misp-galaxy:mitre-malware=\"USBStealer\"", + "user_id": "0" + }, + { + "colour": "#0c9800", + "exportable": true, + "hide_tag": false, + "id": "1011", + "name": "misp-galaxy:tool=\"X-Agent\"", + "user_id": "0" + }, + { + "colour": "#0088cc", + "exportable": true, + "hide_tag": false, + "id": "3013", + "name": "misp-galaxy:mitre-malware=\"XAgentOSX\"", + "user_id": "0" + }, + { + "colour": "#0088cc", + "exportable": true, + "hide_tag": false, + "id": "3014", + "name": "misp-galaxy:mitre-malware=\"CHOPSTICK\"", + "user_id": "0" + }, + { + "colour": "#0088cc", + "exportable": true, + "hide_tag": false, + "id": "3015", + "name": "misp-galaxy:exploit-kit=\"DealersChoice\"", + "user_id": "0" + }, + { + "colour": "#0088cc", + "exportable": true, + "hide_tag": false, + "id": "3016", + "name": "misp-galaxy:mitre-malware=\"Downdelph\"", + "user_id": "0" + } + ], + "analysis": "0", + "attribute_count": "122", + "date": "2017-12-21", + "disable_correlation": false, + "distribution": "3", + "event_creator_email": "alexandre.dulaunoy@circl.lu", + "id": "9747", + "info": "OSINT - Sednit update: How Fancy Bear Spent the Year", + "locked": false, + "org_id": "2", + "orgc_id": "2", + "proposal_email_lock": false, + "publish_timestamp": "0", + "published": false, + "sharing_group_id": "0", + "threat_level_id": "3", + "uuid": "5a3c2fcd-8328-42bb-a95e-4f4402de0b81" + } +} diff --git a/tests/test_mispevent.py b/tests/test_mispevent.py index 919c2cf..3ffe461 100644 --- a/tests/test_mispevent.py +++ b/tests/test_mispevent.py @@ -5,7 +5,7 @@ import unittest import json from io import BytesIO -from pymisp import MISPEvent, MISPSighting +from pymisp import MISPEvent, MISPSighting, MISPTag class TestMISPEvent(unittest.TestCase): @@ -39,6 +39,17 @@ class TestMISPEvent(unittest.TestCase): ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + def test_event_tag(self): + self.init_event() + self.mispevent.add_tag('bar') + self.mispevent.add_tag(name='baz') + new_tag = MISPTag() + new_tag.from_dict(name='foo') + self.mispevent.add_tag(new_tag) + with open('tests/mispevent_testfiles/event_tags.json', 'r') as f: + ref_json = json.load(f) + self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + def test_attribute(self): self.init_event() self.mispevent.add_attribute('filename', 'bar.exe') @@ -152,6 +163,76 @@ class TestMISPEvent(unittest.TestCase): ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + def test_event_not_edited(self): + self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') + self.assertFalse(self.mispevent.edited) + + def test_event_edited(self): + self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') + self.mispevent.info = 'blah' + self.assertTrue(self.mispevent.edited) + + def test_event_tag_edited(self): + self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') + self.assertFalse(self.mispevent.edited) + self.mispevent.add_tag('foo') + self.assertTrue(self.mispevent.edited) + + def test_event_attribute_edited(self): + self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') + self.mispevent.attributes[0].value = 'blah' + self.assertTrue(self.mispevent.attributes[0].edited) + self.assertFalse(self.mispevent.attributes[1].edited) + self.assertTrue(self.mispevent.edited) + + def test_event_attribute_tag_edited(self): + self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') + self.assertFalse(self.mispevent.edited) + self.mispevent.attributes[0].tags[0].name = 'blah' + self.assertTrue(self.mispevent.attributes[0].tags[0].edited) + self.assertFalse(self.mispevent.attributes[0].tags[1].edited) + self.assertTrue(self.mispevent.attributes[0].edited) + self.assertTrue(self.mispevent.edited) + + def test_event_attribute_tag_edited_second(self): + self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') + self.assertFalse(self.mispevent.edited) + self.mispevent.attributes[0].add_tag(name='blah') + self.assertTrue(self.mispevent.attributes[0].edited) + self.assertTrue(self.mispevent.edited) + + def test_event_object_edited(self): + self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') + self.assertFalse(self.mispevent.edited) + self.mispevent.objects[0].comment = 'blah' + self.assertTrue(self.mispevent.objects[0].edited) + self.assertFalse(self.mispevent.objects[1].edited) + self.assertTrue(self.mispevent.edited) + + def test_event_object_attribute_edited(self): + self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') + self.assertFalse(self.mispevent.edited) + self.mispevent.objects[0].attributes[0].comment = 'blah' + self.assertTrue(self.mispevent.objects[0].attributes[0].edited) + self.assertTrue(self.mispevent.objects[0].edited) + self.assertTrue(self.mispevent.edited) + + def test_event_object_attribute_edited_tag(self): + self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') + self.assertFalse(self.mispevent.edited) + self.mispevent.objects[0].attributes[0].add_tag('blah') + self.assertTrue(self.mispevent.objects[0].attributes[0].edited) + self.assertTrue(self.mispevent.objects[0].edited) + self.assertTrue(self.mispevent.edited) + with open('tests/mispevent_testfiles/existing_event_edited.json', 'r') as f: + ref_json = json.load(f) + self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + + def test_obj_by_id(self): + self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') + misp_obj = self.mispevent.get_object_by_id(1556) + self.assertEqual(misp_obj.uuid, '5a3cd604-e11c-4de5-bbbf-c170950d210f') + if __name__ == '__main__': unittest.main() From 96f75cba8ac9791abb55271ac68b6b51aa50daa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 5 Jan 2018 19:24:29 +0100 Subject: [PATCH 107/125] chg: Bump misp-objects --- pymisp/data/misp-objects | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index 875f97d..6027918 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit 875f97dce128a94d53ad13ae85c3d5e413cf1dab +Subproject commit 60279184ddfaf07c37f7ea2e76b5b282a5c7d9c2 From c41281030bc73f3e9fa0e5fa0214e1c4cb180b4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 8 Jan 2018 11:59:32 +0100 Subject: [PATCH 108/125] chg: Move MISPTag to Abstract MISP. --- pymisp/__init__.py | 6 +-- pymisp/abstract.py | 54 ++++++++++++++++++++++ pymisp/exceptions.py | 5 ++- pymisp/mispevent.py | 104 ------------------------------------------- 4 files changed, 61 insertions(+), 108 deletions(-) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index a0e8c5f..8dc23fe 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -27,10 +27,10 @@ def deprecated(func): try: - from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate # noqa + from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate, PyMISPInvalidFormat # noqa from .api import PyMISP # noqa - from .abstract import AbstractMISP, MISPEncode # noqa - from .mispevent import MISPEvent, MISPAttribute, MISPObjectReference, MISPObjectAttribute, MISPObject, MISPTag, MISPUser, MISPOrganisation, MISPSighting # noqa + from .abstract import AbstractMISP, MISPEncode, MISPTag # noqa + from .mispevent import MISPEvent, MISPAttribute, MISPObjectReference, MISPObjectAttribute, MISPObject, MISPUser, MISPOrganisation, MISPSighting # noqa from .tools import AbstractMISPObjectGenerator # noqa from .tools import Neo4j # noqa from .tools import stix # noqa diff --git a/pymisp/abstract.py b/pymisp/abstract.py index 702df34..957f22a 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -10,6 +10,9 @@ import collections import six # Remove that import when discarding python2 support. import logging +from .exceptions import PyMISPInvalidFormat + + logger = logging.getLogger('pymisp') if six.PY2: @@ -49,6 +52,14 @@ class AbstractMISP(collections.MutableMapping): super(AbstractMISP, self).__init__() self.__edited = True # As we create a new object, we assume it is edited + # List of classes having tags + from .mispevent import MISPAttribute, MISPEvent + self.__has_tags = (MISPAttribute, MISPEvent) + if isinstance(self, self.__has_tags): + self.Tag = [] + setattr(AbstractMISP, 'add_tag', AbstractMISP.__add_tag) + setattr(AbstractMISP, 'tags', property(AbstractMISP.__get_tags, AbstractMISP.__set_tags)) + @property def properties(self): """All the class public properties that will be dumped in the dictionary, and the JSON export. @@ -175,3 +186,46 @@ class AbstractMISP(collections.MutableMapping): return int(d.timestamp()) else: return int((d - datetime.datetime.fromtimestamp(0, UTC())).total_seconds()) + + def __add_tag(self, tag=None, **kwargs): + """Add a tag to the attribute (by name or a MISPTag object)""" + if isinstance(tag, str): + misp_tag = MISPTag() + misp_tag.from_dict(name=tag) + elif isinstance(tag, MISPTag): + misp_tag = tag + elif isinstance(tag, dict): + misp_tag = MISPTag() + misp_tag.from_dict(**tag) + elif kwargs: + misp_tag = MISPTag() + misp_tag.from_dict(**kwargs) + else: + raise PyMISPInvalidFormat("The tag is in an invalid format (can be either string, MISPTag, or an expanded dict): {}".format(tag)) + self.Tag.append(misp_tag) + self.edited = True + + def __get_tags(self): + """Returns a lost of tags associated to this Attribute""" + return self.Tag + + def __set_tags(self, tags): + """Set a list of prepared MISPTag.""" + if all(isinstance(x, MISPTag) for x in tags): + self.Tag = tags + else: + raise PyMISPInvalidFormat('All the attributes have to be of type MISPTag.') + + +class MISPTag(AbstractMISP): + def __init__(self): + super(MISPTag, self).__init__() + + def from_dict(self, name, **kwargs): + self.name = name + super(MISPTag, self).from_dict(**kwargs) + + def __repr__(self): + if hasattr(self, 'name'): + return '<{self.__class__.__name__}(name={self.name})'.format(self=self) + return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) diff --git a/pymisp/exceptions.py b/pymisp/exceptions.py index d828e74..967e9b7 100644 --- a/pymisp/exceptions.py +++ b/pymisp/exceptions.py @@ -1,4 +1,3 @@ - # -*- coding: utf-8 -*- @@ -44,3 +43,7 @@ class InvalidMISPObject(MISPObjectException): class UnknownMISPObjectTemplate(MISPObjectException): """Exception raised when the template is unknown""" pass + + +class PyMISPInvalidFormat(PyMISPError): + pass diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index b44bdaf..9c7805a 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -114,19 +114,6 @@ class MISPAttribute(AbstractMISP): return self._malware_binary return None - @property - def tags(self): - """Returns a lost of tags associated to this Attribute""" - return self.Tag - - @tags.setter - def tags(self, tags): - """Set a list of prepared MISPTag.""" - if all(isinstance(x, MISPTag) for x in tags): - self.Tag = tags - else: - raise PyMISPError('All the attributes have to be of type MISPTag.') - @property def shadow_attributes(self): return self.ShadowAttribute @@ -143,24 +130,6 @@ class MISPAttribute(AbstractMISP): """Mark the attribute as deleted (soft delete)""" self.deleted = True - def add_tag(self, tag=None, **kwargs): - """Add a tag to the attribute (by name or a MISPTag object)""" - if isinstance(tag, str): - misp_tag = MISPTag() - misp_tag.from_dict(name=tag) - elif isinstance(tag, MISPTag): - misp_tag = tag - elif isinstance(tag, dict): - misp_tag = MISPTag() - misp_tag.from_dict(**tag) - elif kwargs: - misp_tag = MISPTag() - misp_tag.from_dict(**kwargs) - else: - raise PyMISPError("The tag is in an invalid format (can be either string, MISPTag, or an expanded dict): {}".format(tag)) - self.tags.append(misp_tag) - self.edited = True - def add_proposal(self, shadow_attribute=None, **kwargs): """Alias for add_shadow_attribute""" self.add_shadow_attribute(shadow_attribute, **kwargs) @@ -428,17 +397,6 @@ class MISPEvent(AbstractMISP): else: raise PyMISPError('All the attributes have to be of type MISPObject.') - @property - def tags(self): - return self.Tag - - @tags.setter - def tags(self, tags): - if all(isinstance(x, MISPTag) for x in tags): - self.Tag = tags - else: - raise PyMISPError('All the attributes have to be of type MISPTag.') - def load_file(self, event_path): """Load a JSON dump from a file on the disk""" if not os.path.exists(event_path): @@ -583,24 +541,6 @@ class MISPEvent(AbstractMISP): self.shadow_attributes.append(misp_shadow_attribute) self.edited = True - def add_tag(self, tag=None, **kwargs): - """Add a tag to the attribute (by name or a MISPTag object)""" - if isinstance(tag, str): - misp_tag = MISPTag() - misp_tag.from_dict(name=tag) - elif isinstance(tag, MISPTag): - misp_tag = tag - elif isinstance(tag, dict): - misp_tag = MISPTag() - misp_tag.from_dict(**tag) - elif kwargs: - misp_tag = MISPTag() - misp_tag.from_dict(**kwargs) - else: - raise PyMISPError("The tag is in an invalid format (can be either string, MISPTag, or an expanded dict): {}".format(tag)) - self.tags.append(misp_tag) - self.edited = True - def get_attribute_tag(self, attribute_identifier): '''Return the tags associated to an attribute or an object attribute. :attribute_identifier: can be an ID, UUID, or the value. @@ -768,20 +708,6 @@ class MISPEvent(AbstractMISP): return self.to_dict() -class MISPTag(AbstractMISP): - def __init__(self): - super(MISPTag, self).__init__() - - def from_dict(self, name, **kwargs): - self.name = name - super(MISPTag, self).from_dict(**kwargs) - - def __repr__(self): - if hasattr(self, 'name'): - return '<{self.__class__.__name__}(name={self.name})'.format(self=self) - return '<{self.__class__.__name__}(NotInitialized)'.format(self=self) - - class MISPObjectReference(AbstractMISP): def __init__(self): @@ -993,36 +919,6 @@ class MISPObject(AbstractMISP): self.ObjectReference.append(reference) self.edited = True - # Not supported yet - https://github.com/MISP/PyMISP/issues/168 - # @property - # def tags(self): - # return self.Tag - - # @tags.setter - # def tags(self, tags): - # if all(isinstance(x, MISPTag) for x in tags): - # self.Tag = tags - # else: - # raise PyMISPError('All the attributes have to be of type MISPTag.') - - # def add_tag(self, tag=None, **kwargs): - # """Add a tag to the attribute (by name or a MISPTag object)""" - # if isinstance(tag, str): - # misp_tag = MISPTag() - # misp_tag.from_dict(name=tag) - # elif isinstance(tag, MISPTag): - # misp_tag = tag - # elif isinstance(tag, dict): - # misp_tag = MISPTag() - # misp_tag.from_dict(**tag) - # elif kwargs: - # misp_tag = MISPTag() - # misp_tag.from_dict(**kwargs) - # else: - # raise PyMISPError("The tag is in an invalid format (can be either string, MISPTag, or an expanded dict): {}".format(tag)) - # self.tags.append(misp_tag) - # self.edited = True - def get_attributes_by_relation(self, object_relation): '''Returns the list of attributes with the given object relation in the object''' return self.__fast_attribute_access.get(object_relation, []) From 6c21728e04ea040fe4ac930b7e784f5a874b0744 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 8 Jan 2018 12:09:48 +0100 Subject: [PATCH 109/125] chg: Cleanup from last commit --- pymisp/mispevent.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 9c7805a..1528e04 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -99,7 +99,6 @@ class MISPAttribute(AbstractMISP): self.__category_type_mapping = describe_types['category_type_mappings'] self.__sane_default = describe_types['sane_defaults'] self.__strict = strict - self.Tag = [] self.ShadowAttribute = [] @property @@ -350,7 +349,6 @@ class MISPEvent(AbstractMISP): describe_types = t['result'] self._types = describe_types['types'] - self.Tag = [] self.Attribute = [] self.Object = [] self.RelatedEvent = [] @@ -840,7 +838,6 @@ class MISPObject(AbstractMISP): self.__fast_attribute_access = {} # Hashtable object_relation: [attributes] self.ObjectReference = [] self.Attribute = [] - # self.Tag = [] See https://github.com/MISP/PyMISP/issues/168 if isinstance(default_attributes_parameters, MISPAttribute): # Just make sure we're not modifying an existing MISPAttribute self._default_attributes_parameters = default_attributes_parameters.to_dict() From a78c5cc25c3fa0cd2c34d77fc3128d37e87be66d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 10 Jan 2018 10:39:36 +0100 Subject: [PATCH 110/125] chg: Bump misp-objects --- pymisp/data/misp-objects | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index 6027918..2edd725 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit 60279184ddfaf07c37f7ea2e76b5b282a5c7d9c2 +Subproject commit 2edd72546686a6fee6cc094916e2f8f41ce5e7cc From bfc476ecfc29dac2dade334528388625788abc75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 10 Jan 2018 10:44:37 +0100 Subject: [PATCH 111/125] chg: Bump version --- pymisp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 8dc23fe..2d94d5a 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.4.85' +__version__ = '2.4.85.1' import sys import logging import functools From 85bc693a63af16ad0b7f8ecb1ac7c69848f5cae4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 10 Jan 2018 10:47:20 +0100 Subject: [PATCH 112/125] chg: Bump Changelog --- CHANGELOG.txt | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index f18c409..387b670 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -2,6 +2,71 @@ Changelog ========= +v2.4.85.1 (2018-01-10) +---------------------- + +Changes +~~~~~~~ +- Bump version. [Raphaël Vinot] +- Bump misp-objects. [Raphaël Vinot] +- Cleanup from last commit. [Raphaël Vinot] +- Move MISPTag to Abstract MISP. [Raphaël Vinot] +- Bump misp-objects. [Raphaël Vinot] +- Fix tests (new template version) [Raphaël Vinot] +- Bump misp-objects. [Raphaël Vinot] +- Add test for loading existing malware sample from MISP. [Raphaël + Vinot] +- Multiple changes. [Raphaël Vinot] + + * Fix timestamp dump (properly enforce UTC) + * Properly handle proposals + * Add many getter/setter + * Add dedicated test cases for MISPEvent and other objects +- Allow do pass a category in default_attributes_parameters for object. + [Raphaël Vinot] + + fix #166 +- Default for sharing_group_id is 0. [Raphaël Vinot] +- Add MISPSighting class. [Raphaël Vinot] +- Bump Changelog. [Raphaël Vinot] + +Fix +~~~ +- Edited method works as expected, add tests. [Raphaël Vinot] +- Forgotten test files in last commit... [Raphaël Vinot] +- Disable_correlation from template not properly used. [Raphaël Vinot] +- Don't remove the distribution and sharing_group_id from + default_attributes_parameters. [Raphaël Vinot] +- The sharing_group_id isn't required. [Raphaël Vinot] +- Last commit was broken... [Raphaël Vinot] +- Properly set Tag to attributes within objects. [Raphaël Vinot] +- Add method to add tags to objects. [Raphaël Vinot] + + Fix #160 +- Typo in set_sightings. [Raphaël Vinot] + + Fix #161 + +Other +~~~~~ +- Merge pull request #164 from MISP/refactor. [Raphaël Vinot] + + chg: Multiple changes +- Merge pull request #162 from AninaAntonie/patch-1. [Raphaël Vinot] + + fix: set_sightings +- Set_sightings. [AninaAntonie] + + Maybe I didn't use it correctly but the method set_sightings didn't work for me. It's working now but I'm not sure whether sending a request for every sighting in the list is the best solution. +- Merge pull request #165 from dadokkio/master. [Raphaël Vinot] + + _default_attributes_parameters - if set - is a dict +- _default_attributes_parameters - if set - is a dict. [Arcuri Davide] + + Manage distribution and sharing_group_id as dict key like the other fields. + -- Not sure about default + + v2.4.85 (2017-12-22) -------------------- From baa617aae4a80eae6214c679a9c5736e393c40a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 10 Jan 2018 23:39:27 +0100 Subject: [PATCH 113/125] fix: Download old samples was broken. --- pymisp/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/api.py b/pymisp/api.py index 7c79542..e9b3653 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1147,7 +1147,7 @@ class PyMISP(object): zipped = BytesIO(decoded) try: archive = zipfile.ZipFile(zipped) - if f.get('md5'): + if f.get('md5') and f['md5'] in archive.infolist(): # New format unzipped = BytesIO(archive.open(f['md5'], pwd=b'infected').read()) else: From 89e900671c610d02ec5bf0e3e4d4a82ce7f5fece Mon Sep 17 00:00:00 2001 From: Andras Iklody Date: Thu, 11 Jan 2018 11:58:50 +0100 Subject: [PATCH 114/125] Update settings.default.py --- examples/feed-generator/settings.default.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/feed-generator/settings.default.py b/examples/feed-generator/settings.default.py index b80ba93..47437c0 100755 --- a/examples/feed-generator/settings.default.py +++ b/examples/feed-generator/settings.default.py @@ -16,9 +16,9 @@ outputdir = 'output' # you can use on the event index, such as organisation, tags, etc. # It uses the same joining and condition rules as the API parameters # For example: -# filters = {'tag':'tlp:white|feed-export|!privint','org':'CIRCL'} -# the above would generate a feed for all events created by CIRCL, tagged -# tlp:white and/or feed-export but exclude anything tagged privint +# filters = {'tag':'tlp:white|feed-export|!privint','org':'CIRCL', 'published':1} +# the above would generate a feed for all published events created by CIRCL, +# tagged tlp:white and/or feed-export but exclude anything tagged privint filters = {} From 716fd2723c025dc26de72027005f45bac3263034 Mon Sep 17 00:00:00 2001 From: Tobias Mainka Date: Thu, 11 Jan 2018 12:39:19 +0100 Subject: [PATCH 115/125] added misp object templates path argument --- pymisp/mispevent.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 1528e04..eb3e052 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -814,9 +814,12 @@ class MISPObject(AbstractMISP): super(MISPObject, self).__init__(**kwargs) self.__strict = strict self.name = name - self.__misp_objects_path = os.path.join( - os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)), - 'data', 'misp-objects', 'objects') + if kwargs.get('misp_objects_path', None): + self.__misp_objects_path = kwargs.get('misp_objects_path', None) + else: + self.__misp_objects_path = os.path.join( + os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)), + 'data', 'misp-objects', 'objects') if os.path.exists(os.path.join(self.__misp_objects_path, self.name, 'definition.json')): self.__known_template = True else: From e17349cba2d3db8430dd205128382663d4e1273d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 11 Jan 2018 14:07:31 +0100 Subject: [PATCH 116/125] chg: Remove old warning. --- pymisp/tools/create_misp_object.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pymisp/tools/create_misp_object.py b/pymisp/tools/create_misp_object.py index cc0c6fc..4f6e745 100644 --- a/pymisp/tools/create_misp_object.py +++ b/pymisp/tools/create_misp_object.py @@ -90,6 +90,4 @@ def make_binary_objects(filepath=None, pseudofile=None, filename=None, standalon logger.warning(e) if not HAS_LIEF: logger.warning('Please install lief, documentation here: https://github.com/lief-project/LIEF') - if not filepath: - logger.warning('LIEF currently requires a filepath and not a pseudo file') return misp_file, None, None From 8d9a5af8d3a26a04d1469ac0406129ca2cece67c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 12 Jan 2018 00:35:57 +0100 Subject: [PATCH 117/125] chg: Allow to pass a directory with custom object templates --- pymisp/mispevent.py | 52 ++++++++++--------- .../test_object_template/definition.json | 29 +++++++++++ tests/test_mispevent.py | 22 ++++++++ 3 files changed, 78 insertions(+), 25 deletions(-) create mode 100644 tests/mispevent_testfiles/test_object_template/definition.json diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index eb3e052..90f7cfb 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -10,7 +10,6 @@ from zipfile import ZipFile import hashlib import sys import uuid -from collections import Counter from . import deprecated from .abstract import AbstractMISP @@ -810,17 +809,22 @@ class MISPObject(AbstractMISP): In this case the ObjectReference needs to be pushed manually and cannot be in the JSON dump. :default_attributes_parameters: Used as template for the attributes if they are not overwritten in add_attribute + + :misp_objects_path_custom: Path to custom object templates ''' super(MISPObject, self).__init__(**kwargs) self.__strict = strict self.name = name - if kwargs.get('misp_objects_path', None): - self.__misp_objects_path = kwargs.get('misp_objects_path', None) - else: - self.__misp_objects_path = os.path.join( - os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)), - 'data', 'misp-objects', 'objects') - if os.path.exists(os.path.join(self.__misp_objects_path, self.name, 'definition.json')): + misp_objects_path = os.path.join( + os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)), + 'data', 'misp-objects', 'objects') + misp_objects_path_custom = kwargs.get('misp_objects_path_custom') + if misp_objects_path_custom and os.path.exists(os.path.join(misp_objects_path_custom, self.name, 'definition.json')): + # Use the local object path by default if provided (allows to overwrite a default template) + template_path = os.path.join(misp_objects_path_custom, self.name, 'definition.json') + self.__known_template = True + elif os.path.exists(os.path.join(misp_objects_path, self.name, 'definition.json')): + template_path = os.path.join(misp_objects_path, self.name, 'definition.json') self.__known_template = True else: if self.__strict: @@ -828,7 +832,7 @@ class MISPObject(AbstractMISP): else: self.__known_template = False if self.__known_template: - with open(os.path.join(self.__misp_objects_path, self.name, 'definition.json'), 'r') as f: + with open(template_path, 'r') as f: self.__definition = json.load(f) setattr(self, 'meta-category', self.__definition['meta-category']) self.template_uuid = self.__definition['uuid'] @@ -962,23 +966,21 @@ class MISPObject(AbstractMISP): def _validate(self): """Make sure the object we're creating has the required fields""" - all_object_relations = [] - for a in self.attributes: - all_object_relations.append(a.object_relation) - count_relations = dict(Counter(all_object_relations)) - for key, counter in count_relations.items(): - if counter == 1: - continue - if not self.__definition['attributes'][key].get('multiple'): - raise InvalidMISPObject('Multiple occurrences of {} is not allowed'.format(key)) - all_attribute_names = set(count_relations.keys()) - if self.__definition.get('requiredOneOf'): - if not set(self.__definition['requiredOneOf']) & all_attribute_names: - raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self.__definition['requiredOneOf']))) if self.__definition.get('required'): - for r in self.__definition.get('required'): - if r not in all_attribute_names: - raise InvalidMISPObject('{} is required'.format(r)) + required_missing = set(self.__definition.get('required')) - set(self.__fast_attribute_access.keys()) + if required_missing: + raise InvalidMISPObject('{} are required.'.format(required_missing)) + if self.__definition.get('requiredOneOf'): + if not set(self.__definition['requiredOneOf']) & set(self.__fast_attribute_access.keys()): + # We ecpect at least one of the object_relation in requiredOneOf, and it isn't the case + raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self.__definition['requiredOneOf']))) + for rel, attrs in self.__fast_attribute_access.items(): + if len(attrs) == 1: + # object_relation's here only once, everything's cool, moving on + continue + if not self.__definition['attributes'][rel].get('multiple'): + # object_relation's here more than once, but it isn't allowed in the template. + raise InvalidMISPObject('Multiple occurrences of {} is not allowed'.format(rel)) return True def __repr__(self): diff --git a/tests/mispevent_testfiles/test_object_template/definition.json b/tests/mispevent_testfiles/test_object_template/definition.json new file mode 100644 index 0000000..283b9de --- /dev/null +++ b/tests/mispevent_testfiles/test_object_template/definition.json @@ -0,0 +1,29 @@ +{ + "requiredOneOf": [ + "member1", + "member2" + ], + "required": [ + "member3" + ], + "attributes": { + "member1": { + "description": "FirstMember", + "misp-attribute": "text" + }, + "member2": { + "description": "SecondMember", + "misp-attribute": "text", + "multiple": true + }, + "member3": { + "description": "Thirdmember", + "misp-attribute": "text" + } + }, + "version": 1, + "description": "TestTemplate.", + "meta-category": "file", + "uuid": "4ec55cc6-9e49-4c64-b794-03c25c1a6589", + "name": "test_object_template" +} diff --git a/tests/test_mispevent.py b/tests/test_mispevent.py index 3ffe461..0921de1 100644 --- a/tests/test_mispevent.py +++ b/tests/test_mispevent.py @@ -6,6 +6,7 @@ import json from io import BytesIO from pymisp import MISPEvent, MISPSighting, MISPTag +from pymisp.exceptions import InvalidMISPObject class TestMISPEvent(unittest.TestCase): @@ -233,6 +234,27 @@ class TestMISPEvent(unittest.TestCase): misp_obj = self.mispevent.get_object_by_id(1556) self.assertEqual(misp_obj.uuid, '5a3cd604-e11c-4de5-bbbf-c170950d210f') + def test_userdefined_object(self): + self.init_event() + self.mispevent.add_object(name='test_object_template', strict=True, misp_objects_path_custom='tests/mispevent_testfiles') + with self.assertRaises(InvalidMISPObject) as e: + # Fail on required + self.mispevent.to_json() + self.assertEqual(e.exception.message, '{\'member3\'} are required.') + + self.mispevent.objects[0].add_attribute('member3', value='foo') + with self.assertRaises(InvalidMISPObject) as e: + # Fail on requiredOneOf + self.mispevent.to_json() + self.assertEqual(e.exception.message, 'At least one of the following attributes is required: member1, member2') + + self.mispevent.objects[0].add_attribute('member1', value='bar') + self.mispevent.objects[0].add_attribute('member1', value='baz') + with self.assertRaises(InvalidMISPObject) as e: + # member1 is not a multiple + self.mispevent.to_json() + self.assertEqual(e.exception.message, 'Multiple occurrences of member1 is not allowed') + if __name__ == '__main__': unittest.main() From 4e36ac34dc391780ca0e063a71adc89603242ff5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 12 Jan 2018 00:45:34 +0100 Subject: [PATCH 118/125] fix: Make python2 happy. --- tests/test_mispevent.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_mispevent.py b/tests/test_mispevent.py index 0921de1..8de1180 100644 --- a/tests/test_mispevent.py +++ b/tests/test_mispevent.py @@ -3,6 +3,7 @@ import unittest import json +import sys from io import BytesIO from pymisp import MISPEvent, MISPSighting, MISPTag @@ -240,7 +241,11 @@ class TestMISPEvent(unittest.TestCase): with self.assertRaises(InvalidMISPObject) as e: # Fail on required self.mispevent.to_json() - self.assertEqual(e.exception.message, '{\'member3\'} are required.') + if sys.version_info >= (3, ): + self.assertEqual(e.exception.message, '{\'member3\'} are required.') + else: + # Python2 bullshit + self.assertEqual(e.exception.message, 'set([u\'member3\']) are required.') self.mispevent.objects[0].add_attribute('member3', value='foo') with self.assertRaises(InvalidMISPObject) as e: From b733ddbdb00d72a7d195c9c588c14af42b16dd82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 12 Jan 2018 16:15:09 +0100 Subject: [PATCH 119/125] chg: Improve Object Attribute editing --- pymisp/mispevent.py | 36 ++++++++++++++--- .../mispevent_testfiles/misp_custom_obj.json | 40 +++++++++++++++++++ tests/test_mispevent.py | 6 +++ 3 files changed, 76 insertions(+), 6 deletions(-) create mode 100644 tests/mispevent_testfiles/misp_custom_obj.json diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 90f7cfb..4eb90b1 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -878,10 +878,25 @@ class MISPObject(AbstractMISP): def attributes(self): return self.Attribute + @attributes.setter + def attributes(self, attributes): + if all(isinstance(x, MISPObjectAttribute) for x in attributes): + self.Attribute = attributes + self.__fast_attribute_access = {} + else: + raise PyMISPError('All the attributes have to be of type MISPObjectAttribute.') + @property def references(self): return self.ObjectReference + @references.setter + def references(self, references): + if all(isinstance(x, MISPObjectReference) for x in references): + self.ObjectReference = references + else: + raise PyMISPError('All the attributes have to be of type MISPObjectReference.') + def from_dict(self, **kwargs): if self.__known_template: if kwargs.get('template_uuid') and kwargs['template_uuid'] != self.template_uuid: @@ -925,11 +940,20 @@ class MISPObject(AbstractMISP): def get_attributes_by_relation(self, object_relation): '''Returns the list of attributes with the given object relation in the object''' - return self.__fast_attribute_access.get(object_relation, []) + return self._fast_attribute_access.get(object_relation, []) + + @property + def _fast_attribute_access(self): + if not self.__fast_attribute_access: + for a in self.attributes: + if not self.__fast_attribute_access.get(a.object_relation): + self.__fast_attribute_access[a.object_relation] = [] + self.__fast_attribute_access[a.object_relation].append(a) + return self.__fast_attribute_access def has_attributes_by_relation(self, list_of_relations): '''True if all the relations in the list are defined in the object''' - return all(relation in self.__fast_attribute_access for relation in list_of_relations) + return all(relation in self._fast_attribute_access for relation in list_of_relations) def add_attribute(self, object_relation, **value): """Add an attribute. object_relation is required and the value key is a @@ -949,8 +973,8 @@ class MISPObject(AbstractMISP): attribute.from_dict(object_relation=object_relation, **dict(self._default_attributes_parameters, **value)) if not self.__fast_attribute_access.get(object_relation): self.__fast_attribute_access[object_relation] = [] - self.Attribute.append(attribute) self.__fast_attribute_access[object_relation].append(attribute) + self.Attribute.append(attribute) self.edited = True return attribute @@ -967,14 +991,14 @@ class MISPObject(AbstractMISP): def _validate(self): """Make sure the object we're creating has the required fields""" if self.__definition.get('required'): - required_missing = set(self.__definition.get('required')) - set(self.__fast_attribute_access.keys()) + required_missing = set(self.__definition.get('required')) - set(self._fast_attribute_access.keys()) if required_missing: raise InvalidMISPObject('{} are required.'.format(required_missing)) if self.__definition.get('requiredOneOf'): - if not set(self.__definition['requiredOneOf']) & set(self.__fast_attribute_access.keys()): + if not set(self.__definition['requiredOneOf']) & set(self._fast_attribute_access.keys()): # We ecpect at least one of the object_relation in requiredOneOf, and it isn't the case raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self.__definition['requiredOneOf']))) - for rel, attrs in self.__fast_attribute_access.items(): + for rel, attrs in self._fast_attribute_access.items(): if len(attrs) == 1: # object_relation's here only once, everything's cool, moving on continue diff --git a/tests/mispevent_testfiles/misp_custom_obj.json b/tests/mispevent_testfiles/misp_custom_obj.json new file mode 100644 index 0000000..024fd82 --- /dev/null +++ b/tests/mispevent_testfiles/misp_custom_obj.json @@ -0,0 +1,40 @@ +{ + "Event": { + "Object": [ + { + "Attribute": [ + { + "category": "Other", + "disable_correlation": false, + "object_relation": "member3", + "to_ids": false, + "type": "text", + "value": "foo" + }, + { + "category": "Other", + "disable_correlation": false, + "object_relation": "member1", + "to_ids": false, + "type": "text", + "value": "bar" + } + ], + "description": "TestTemplate.", + "distribution": 5, + "meta-category": "file", + "misp_objects_path_custom": "tests/mispevent_testfiles", + "name": "test_object_template", + "sharing_group_id": 0, + "template_uuid": "4ec55cc6-9e49-4c64-b794-03c25c1a6589", + "template_version": 1, + "uuid": "a" + } + ], + "analysis": "1", + "date": "2017-12-31", + "distribution": "1", + "info": "This is a test", + "threat_level_id": "1" + } +} diff --git a/tests/test_mispevent.py b/tests/test_mispevent.py index 8de1180..4472ad2 100644 --- a/tests/test_mispevent.py +++ b/tests/test_mispevent.py @@ -260,6 +260,12 @@ class TestMISPEvent(unittest.TestCase): self.mispevent.to_json() self.assertEqual(e.exception.message, 'Multiple occurrences of member1 is not allowed') + self.mispevent.objects[0].attributes = self.mispevent.objects[0].attributes[:2] + self.mispevent.objects[0].uuid = 'a' + with open('tests/mispevent_testfiles/misp_custom_obj.json', 'r') as f: + ref_json = json.load(f) + self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + if __name__ == '__main__': unittest.main() From cb4f4d2443d292b83aa6cef8496f025a8dc2c502 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 12 Jan 2018 16:15:38 +0100 Subject: [PATCH 120/125] chg: Bump misp-objects --- pymisp/data/misp-objects | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index 2edd725..21e58b3 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit 2edd72546686a6fee6cc094916e2f8f41ce5e7cc +Subproject commit 21e58b3ddf1737028b556b93b20d848f86a71cd0 From c862800df20350cc6e18b73e7fcb98f5e43593bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 12 Jan 2018 18:47:45 +0100 Subject: [PATCH 121/125] fix: add_hashes was broken Fix #174 --- pymisp/api.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index e9b3653..d4210e7 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -17,7 +17,7 @@ import zipfile from . import __version__, deprecated from .exceptions import PyMISPError, SearchError, NoURL, NoKey from .mispevent import MISPEvent, MISPAttribute, MISPUser, MISPOrganisation, MISPSighting -from .abstract import MISPEncode +from .abstract import AbstractMISP, MISPEncode logger = logging.getLogger('pymisp') @@ -478,15 +478,18 @@ class PyMISP(object): else: url = urljoin(self.root_url, 'attributes/add/{}'.format(event_id)) if isinstance(attributes, list): - values = [] - for a in attributes: - values.append(a['value']) - attributes[0]['value'] = values - data = attributes[0].to_json() + if all(isinstance(a, AbstractMISP) for a in attributes): + data = attributes + else: + values = [] + for a in attributes: + values.append(a['value']) + attributes[0]['value'] = values + data = attributes[0].to_json() else: data = attributes.to_json() # __prepare_request(...) returns a requests.Response Object - responses.append(self.__prepare_request('POST', url, data).json()) + responses.append(self.__prepare_request('POST', url, json.dumps(data, cls=MISPEncode)).json()) return responses def _extract_event_id(self, event): From f122cdca90290fdae2381072e87868b0410ee268 Mon Sep 17 00:00:00 2001 From: Louis LCE Date: Fri, 12 Jan 2018 22:01:08 +0100 Subject: [PATCH 122/125] Fix add_hashes test function parameters --- tests/test.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/test.py b/tests/test.py index acd5a77..08a7a2a 100755 --- a/tests/test.py +++ b/tests/test.py @@ -54,7 +54,17 @@ class TestBasic(unittest.TestCase): def add_hashes(self, eventid): r = self.misp.get_event(eventid) event = r.json() - event = self.misp.add_hashes(event, 'Payload installation', 'dll_installer.dll', '0a209ac0de4ac033f31d6ba9191a8f7a', '1f0ae54ac3f10d533013f74f48849de4e65817a7', '003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9', 'Fanny modules', False, 2) + event = self.misp.add_hashes(event, + category='Payload installation', + filename='dll_installer.dll', + md5='0a209ac0de4ac033f31d6ba9191a8f7a', + sha1='1f0ae54ac3f10d533013f74f48849de4e65817a7', + sha256='003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9', + ssdeep=None, + comment='Fanny modules', + to_ids=False, + distribution=2, + proposal=False) self._clean_event(event) to_check = {u'Event': {u'info': u'This is a test', u'locked': False, u'attribute_count': u'3', u'analysis': u'0', From aff57876bcacbec5a910fcf815cfdd7442bd93c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 16 Jan 2018 12:15:30 +0100 Subject: [PATCH 123/125] chg: Raise an exception when distribution is sharing group, but the ID is missing. --- pymisp/mispevent.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 4eb90b1..e59adc4 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -203,6 +203,15 @@ class MISPAttribute(AbstractMISP): self.timestamp = datetime.datetime.fromtimestamp(int(kwargs.pop('timestamp')), UTC()) if kwargs.get('sharing_group_id'): self.sharing_group_id = int(kwargs.pop('sharing_group_id')) + + if self.distribution == 4: + # The distribution is set to sharing group, a sharing_group_id is required. + if not hasattr(self, 'sharing_group_id'): + raise NewAttributeError('If the distribution is set to sharing group, a sharing group ID is required.') + elif not self.sharing_group_id: + # Cannot be None or 0 either. + raise NewAttributeError('If the distribution is set to sharing group, a sharing group ID is required (cannot be {}).'.format(self.sharing_group_id)) + if kwargs.get('Tag'): for tag in kwargs.pop('Tag'): self.add_tag(tag) From aaf18fa3c8fb1362143476c5e14dd8749ece7cd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arm=C4=ABns=20Palms?= Date: Thu, 18 Jan 2018 15:17:52 +0200 Subject: [PATCH 124/125] Change the comment of attribute --- pymisp/api.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pymisp/api.py b/pymisp/api.py index d4210e7..a0b5f52 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -902,6 +902,11 @@ class PyMISP(object): query = {"to_ids": to_ids} return self.__query('edit/{}'.format(attribute_uuid), query, controller='attributes') + def change_comment(self, attribute_uuid, comment): + """Change the comment of attribute""" + query = {"comment": comment} + return self.__query('edit/{}'.format(attribute_uuid), query, controller='attributes') + # ############################## # ###### Attribute update ###### # ############################## From 9158985f3f22146e04d0e2d590e16033b0346d45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arm=C4=ABns=20Palms?= Date: Fri, 19 Jan 2018 16:57:40 +0200 Subject: [PATCH 125/125] Change in new_tag function. Added attributie 'hide_tag' --- pymisp/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index a0b5f52..8aa15ac 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1218,9 +1218,9 @@ class PyMISP(object): to_return.append(tag['name']) return to_return - def new_tag(self, name=None, colour="#00ace6", exportable=False): + def new_tag(self, name=None, colour="#00ace6", exportable=False, hide_tag=False): """Create a new tag""" - to_post = {'Tag': {'name': name, 'colour': colour, 'exportable': exportable}} + to_post = {'Tag': {'name': name, 'colour': colour, 'exportable': exportable, 'hide_tag': hide_tag}} url = urljoin(self.root_url, 'tags/add') response = self.__prepare_request('POST', url, json.dumps(to_post)) return self._check_response(response)