From 1d4261fa5a393d08d1ee7d362c0c1bc51270109c Mon Sep 17 00:00:00 2001 From: Iglocska Date: Mon, 11 Apr 2016 15:18:05 +0200 Subject: [PATCH 1/4] Added the option to filter out attributes based on distribution level --- examples/feed-generator/generate.py | 19 ++++++++++++++++++- .../{settings.py => settings.default.py} | 16 ++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) rename examples/feed-generator/{settings.py => settings.default.py} (58%) diff --git a/examples/feed-generator/generate.py b/examples/feed-generator/generate.py index 13229b9..361fed3 100755 --- a/examples/feed-generator/generate.py +++ b/examples/feed-generator/generate.py @@ -5,7 +5,7 @@ import sys import json import os from pymisp import PyMISP -from settings import url, key, ssl, outputdir, filters +from settings import * objectsToSave = { @@ -29,8 +29,16 @@ fieldsToSave = ['uuid', 'info', 'threat_level_id', 'analysis', 'timestamp', 'publish_timestamp', 'published', 'date'] +valid_attribute_distributions = [] + def init(): + # If we have an old settings.py file then this variable won't exist + global valid_attribute_distributions + try: + valid_attribute_distributions = valid_attribute_distribution_levels + except: + valid_attribute_distributions = ['0', '1', '2', '3', '4', '5'] return PyMISP(url, key, ssl, 'json') @@ -61,11 +69,20 @@ def __cleanupEventFields(event, temp): return event +def __blockAttributeByDistribution(attribute): + if attribute['distribution'] not in valid_attribute_distributions: + return True + return False + + def __cleanupEventObjects(event, temp): for objectType in objectsToSave.keys(): if objectsToSave[objectType]['multiple'] is True: if objectType in temp['Event']: for objectInstance in temp['Event'][objectType]: + if objectType is 'Attribute': + if __blockAttributeByDistribution(objectInstance): + continue tempObject = {} for field in objectsToSave[objectType]['fields']: if field in objectInstance.keys(): diff --git a/examples/feed-generator/settings.py b/examples/feed-generator/settings.default.py similarity index 58% rename from examples/feed-generator/settings.py rename to examples/feed-generator/settings.default.py index 7901a87..b80ba93 100755 --- a/examples/feed-generator/settings.py +++ b/examples/feed-generator/settings.default.py @@ -21,3 +21,19 @@ outputdir = 'output' # tlp:white and/or feed-export but exclude anything tagged privint filters = {} + +# By default all attributes will be included in the feed generation +# Remove the levels that you do not wish to include in the feed +# Use this to further narrow down what gets exported, for example: +# Setting this to ['3', '5'] will exclude any attributes from the feed that +# are not exportable to all or inherit the event +# +# The levels are as follows: +# 0: Your Organisation Only +# 1: This Community Only +# 2: Connected Communities +# 3: All +# 4: Sharing Group +# 5: Inherit Event +valid_attribute_distribution_levels = ['0', '1', '2', '3', '4', '5'] + From c523e8acf624f42780deb90e91d2538ecdbe49a1 Mon Sep 17 00:00:00 2001 From: Nick Driver Date: Tue, 12 Apr 2016 13:42:01 -0400 Subject: [PATCH 2/4] Add SSDEEP and FILENAME|SSDEEP support --- pymisp/api.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index c04e77d..d1e6094 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -109,8 +109,8 @@ class PyMISP(object): 'Payload delivery', 'Payload installation', 'Artifacts dropped', 'Persistence mechanism', 'Network activity', 'Payload type', 'Attribution', 'External analysis', 'Other'] - self.types = ['md5', 'sha1', 'sha256', 'filename', 'filename|md5', 'filename|sha1', - 'filename|sha256', 'ip-src', 'ip-dst', 'hostname', 'domain', 'url', + self.types = ['md5', 'sha1', 'sha256', 'ssdeep', 'filename', 'filename|md5', 'filename|sha1', + 'filename|sha256', 'filename|ssdeep', 'ip-src', 'ip-dst', 'hostname', 'domain', 'url', 'user-agent', 'http-method', 'regkey', 'regkey|value', 'AS', 'snort', 'pattern-in-file', 'pattern-in-traffic', 'pattern-in-memory', 'named pipe', 'mutex', 'vulnerability', 'attachment', 'malware-sample', 'link', 'comment', @@ -388,7 +388,7 @@ class PyMISP(object): response = self.update_event(event['Event']['id'], event, 'json') return self._check_response(response) - def add_hashes(self, event, category='Artifacts dropped', filename=None, md5=None, sha1=None, sha256=None, comment=None, to_ids=True, distribution=None, proposal=False): + def add_hashes(self, event, category='Artifacts dropped', filename=None, md5=None, sha1=None, sha256=None, ssdeep=None, comment=None, to_ids=True, distribution=None, proposal=False): categories = ['Payload delivery', 'Artifacts dropped', 'Payload installation', 'External analysis'] if category not in categories: raise NewAttributeError('{} is invalid, category has to be in {}'.format(category, (', '.join(categories)))) @@ -408,6 +408,10 @@ class PyMISP(object): if sha256: attributes.append(self._prepare_full_attribute(category, type_value.format('sha256'), value.format(sha256), to_ids, comment, distribution)) + if ssdeep: + attributes.append(self._prepare_full_attribute(category, type_value.format('ssdeep'), value.format(ssdeep), + to_ids, comment, distribution)) + return self._send_attributes(event, attributes, proposal) From c7b0b3df64deb8761ba18d4437fa7faf2ff4653e Mon Sep 17 00:00:00 2001 From: Nick Driver Date: Wed, 13 Apr 2016 15:40:31 -0400 Subject: [PATCH 3/4] Add internal reference attributes --- pymisp/api.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/pymisp/api.py b/pymisp/api.py index c04e77d..cc63f86 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -553,6 +553,28 @@ class PyMISP(object): attributes = [] attributes.append(self._prepare_full_attribute('Attribution', 'threat-actor', target, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) + + # ##### Internal reference attributes ##### + + def add_internal_link(self, event, reference, to_ids=False, comment=None, distribution=None, proposal=False): + attributes = [] + attributes.append(self._prepare_full_attribute('Internal reference', 'link', reference, to_ids, comment, distribution)) + return self._send_attributes(event, attributes, proposal) + + def add_internal_comment(self, event, reference, to_ids=False, comment=None, distribution=None, proposal=False): + attributes = [] + attributes.append(self._prepare_full_attribute('Internal reference', 'comment', reference, to_ids, comment, distribution)) + return self._send_attributes(event, attributes, proposal) + + def add_internal_text(self, event, reference, to_ids=False, comment=None, distribution=None, proposal=False): + attributes = [] + attributes.append(self._prepare_full_attribute('Internal reference', 'text', reference, to_ids, comment, distribution)) + return self._send_attributes(event, attributes, proposal) + + def add_internal_other(self, event, reference, to_ids=False, comment=None, distribution=None, proposal=False): + attributes = [] + attributes.append(self._prepare_full_attribute('Internal reference', 'other', reference, to_ids, comment, distribution)) + return self._send_attributes(event, attributes, proposal) # ################################################## # ######### Upload samples through the API ######### From d493ff76baa5c6f4fe37fda5be2ac859ea4aee90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Thu, 14 Apr 2016 10:29:36 +0200 Subject: [PATCH 4/4] type-category association checking automated --- pymisp/api.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index f261222..c221028 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -105,6 +105,19 @@ class PyMISP(object): self.out_type = out_type self.debug = debug + try: + # Make sure the MISP instance is working and the URL is valid + self.get_version() + except Exception as e: + raise PyMISPError('Unable to connect to MISP ({}). Please make sure the API key and the URL are correct (http/https is required): {}'.format(self.root_url, e)) + + session = self.__prepare_session(out_type) + self.describe_types = session.get(self.root_url + 'attributes/describeTypes.json').json() + + self.categories = self.describe_types['result']['categories'] + self.types = self.describe_types['result']['types'] + self.category_type_mapping = self.describe_types['result']['category_type_mappings'] + self.categories = ['Internal reference', 'Targeting data', 'Antivirus detection', 'Payload delivery', 'Payload installation', 'Artifacts dropped', 'Persistence mechanism', 'Network activity', 'Payload type', @@ -118,11 +131,7 @@ class PyMISP(object): 'yara', 'target-user', 'target-email', 'target-machine', 'target-org', 'target-location', 'target-external', 'other', 'threat-actor'] - try: - # Make sure the MISP instance is working and the URL is valid - self.get_version() - except Exception as e: - raise PyMISPError('Unable to connect to MISP ({}). Please make sure the API key and the URL are correct (http/https is required): {}'.format(self.root_url, e)) + def __prepare_session(self, force_out=None): """ @@ -296,11 +305,14 @@ class PyMISP(object): to_return = {} if category not in self.categories: raise NewAttributeError('{} is invalid, category has to be in {}'.format(category, (', '.join(self.categories)))) - to_return['category'] = category if type_value not in self.types: raise NewAttributeError('{} is invalid, type_value has to be in {}'.format(type_value, (', '.join(self.types)))) + + if type_value not in self.category_type_mapping[category]: + raise NewAttributeError('{} and {} is an invalid combinaison, type_value for this category has to be in {}'.format(type_value, category, (', '.join(self.category_type_mapping[category])))) to_return['type'] = type_value + to_return['category'] = category if to_ids not in [True, False]: raise NewAttributeError('{} is invalid, to_ids has to be True or False'.format(to_ids))