From a67a90d1d31faccc8cdaee64f3fc1365e03c7df7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Larinier?= Date: Wed, 9 Mar 2016 18:37:27 +0100 Subject: [PATCH 001/223] add method to export txt all attributes by type --- pymisp/api.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 8c95c25..023ddcb 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -202,7 +202,7 @@ class PyMISP(object): url = urljoin(self.root_url, 'events/index') if filters is not None: filters = json.dumps(filters) - print filters + print(filters) return session.post(url, data=filters) else: return session.get(url) @@ -862,9 +862,15 @@ class PyMISP(object): return {'version': '{}.{}.{}'.format(master_version['major'], master_version['minor'], master_version['hotfix'])} else: return {'error': 'Impossible to retrieve the version of the master branch.'} + # ############## Export Attributes in text #################################### + def get_all_attributes_txt(self, type_attr): + + session = self.__prepare_session('txt') + url = urljoin(self.root_url,'attributes/text/download/%s' % type_attr) + response = session.get(url) + return response # ############## Deprecated (Pure XML API should not be used) ################## - @deprecated def download_all(self): """ From 39c06a7d245838019dceb8b32fd279f3fee1b294 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Larinier?= Date: Fri, 11 Mar 2016 16:53:31 +0100 Subject: [PATCH 002/223] add add_tag method to an event and value 5 to distribution attribute --- pymisp/api.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pymisp/api.py b/pymisp/api.py index 023ddcb..087f7f5 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -233,6 +233,7 @@ class PyMISP(object): else: return session.post(url, data=event) + def update_event(self, event_id, event, force_out=None): """ Update an event @@ -265,6 +266,7 @@ class PyMISP(object): url = urljoin(self.root_url, 'attributes/{}'.format(attribute_id)) return session.delete(url) + # ############################################## # ######### Event handling (Json only) ######### # ############################################## @@ -303,7 +305,7 @@ class PyMISP(object): if distribution is not None: distribution = int(distribution) # If None: take the default value of the event - if distribution not in [None, 0, 1, 2, 3]: + if distribution not in [None, 0, 1, 2, 3,5]: raise NewAttributeError('{} is invalid, the distribution has to be in 0, 1, 2, 3 or None'.format(distribution)) if distribution is not None: to_return['distribution'] = distribution @@ -354,6 +356,13 @@ class PyMISP(object): response = self.update_event(event['Event']['id'], event, 'json') return self._check_response(response) + def add_tag(self,event, tag): + session = self.__prepare_session('json') + to_post = {'request': {'Event':{'id': event['Event']['id'], 'tag': tag}}} + response = session.post(urljoin(self.root_url, 'events/addTag'), data=json.dumps(to_post)) + + return self._check_response(response) + # ##### File attributes ##### def _send_attributes(self, event, attributes, proposal=False): From 3c90e25ebd5a0b9b0195aedbd2c1c8f91eb97d0c Mon Sep 17 00:00:00 2001 From: Thomas King Date: Mon, 14 Mar 2016 11:17:53 +0000 Subject: [PATCH 003/223] Add threat actor through API, Create new tag --- pymisp/api.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/pymisp/api.py b/pymisp/api.py index 087f7f5..db4908c 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -112,7 +112,7 @@ class PyMISP(object): 'mutex', 'vulnerability', 'attachment', 'malware-sample', 'link', 'comment', 'text', 'email-src', 'email-dst', 'email-subject', 'email-attachment', 'yara', 'target-user', 'target-email', 'target-machine', 'target-org', - 'target-location', 'target-external', 'other'] + 'target-location', 'target-external', 'other', 'threat-actor'] try: # Make sure the MISP instance is working and the URL is valid @@ -536,6 +536,13 @@ class PyMISP(object): attributes.append(self._prepare_full_attribute('Targeting data', 'target-external', target, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) + # ##### Attribution attributes ##### + + def add_threat_actor(self, event, target, to_ids=True, comment=None, distribution=None, proposal=False): + attributes = [] + attributes.append(self._prepare_full_attribute('Attribution', 'threat-actor', target, to_ids, comment, distribution)) + return self._send_attributes(event, attributes, proposal) + # ################################################## # ######### Upload samples through the API ######### # ################################################## @@ -833,6 +840,13 @@ class PyMISP(object): to_return.append(tag['name']) return to_return + def new_tag(self,name=None, colour="#00ace6", exportable=False): + to_post = {'Tag': {'name':name,'colour':colour, 'exportable':exportable}} + session = self.__prepare_session('json') + url = urljoin(self.root_url, 'tags/add') + response = session.post(url, data=json.dumps(to_post)) + return self._check_response(response) + # ########## Version ########## def get_api_version(self): From ba14cf1aaae2c87579e44049ab5959c5763d15fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Larinier?= Date: Tue, 15 Mar 2016 17:17:04 +0100 Subject: [PATCH 004/223] add method change_threat_level --- pymisp/api.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pymisp/api.py b/pymisp/api.py index 087f7f5..6f5f49e 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -363,6 +363,13 @@ class PyMISP(object): return self._check_response(response) + def change_threat_level(self, event, threat_level_id): + event['Event']['threat_level_id'] = threat_level_id + self._prepare_update(event) + response = self.update_event(event['Event']['id'], event) + + return self._check_response(response) + # ##### File attributes ##### def _send_attributes(self, event, attributes, proposal=False): From c73becee36c5ae68788e12318dbeba46e7779dec Mon Sep 17 00:00:00 2001 From: Thomas King Date: Fri, 18 Mar 2016 08:38:04 +0000 Subject: [PATCH 005/223] Added in searchable indexes, only brings back index and count etc, not results within the index --- pymisp/api.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pymisp/api.py b/pymisp/api.py index 78e9f44..adf07fa 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -8,6 +8,7 @@ import datetime import os import base64 import re +import urllib try: from urllib.parse import urljoin @@ -170,6 +171,8 @@ class PyMISP(object): raise PyMISPError('Unknown error: {}'.format(response.text)) errors = [] + if type(to_return) is list: + to_return = {'response':to_return} if to_return.get('error'): if not isinstance(to_return['error'], list): errors.append(to_return['error']) @@ -677,6 +680,13 @@ class PyMISP(object): response = session.post(url, data=json.dumps(query)) return self._check_response(response) + def search_index(self, value): + value = urllib.quote(value) + session = self.__prepare_session('json') + url = urljoin(self.root_url, 'events/index/searchall:%s' % value) + response = session.get(url) + return self._check_response(response) + def search_all(self, value): query = {'value': value, 'searchall': 1} session = self.__prepare_session('json') From 9e8f81aafed4faea6e7d780976c9d05cc431194b Mon Sep 17 00:00:00 2001 From: Thomas King Date: Fri, 18 Mar 2016 08:47:30 +0000 Subject: [PATCH 006/223] Python 2/3 compatible, urllib module --- pymisp/api.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index adf07fa..fc213c5 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -8,11 +8,12 @@ import datetime import os import base64 import re -import urllib try: from urllib.parse import urljoin + from urllib.parse import quote except ImportError: + from urllib import quote from urlparse import urljoin from io import BytesIO import zipfile @@ -681,7 +682,7 @@ class PyMISP(object): return self._check_response(response) def search_index(self, value): - value = urllib.quote(value) + value = quote(value) session = self.__prepare_session('json') url = urljoin(self.root_url, 'events/index/searchall:%s' % value) response = session.get(url) From a0c91e5c37982ca59e2847432d384797c8464b48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 21 Mar 2016 14:55:41 +0100 Subject: [PATCH 007/223] Make pep8 happy --- pymisp/api.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 78e9f44..1ea3b45 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -233,7 +233,6 @@ class PyMISP(object): else: return session.post(url, data=event) - def update_event(self, event_id, event, force_out=None): """ Update an event @@ -266,7 +265,6 @@ class PyMISP(object): url = urljoin(self.root_url, 'attributes/{}'.format(attribute_id)) return session.delete(url) - # ############################################## # ######### Event handling (Json only) ######### # ############################################## @@ -305,8 +303,8 @@ class PyMISP(object): if distribution is not None: distribution = int(distribution) # If None: take the default value of the event - if distribution not in [None, 0, 1, 2, 3,5]: - raise NewAttributeError('{} is invalid, the distribution has to be in 0, 1, 2, 3 or None'.format(distribution)) + if distribution not in [None, 0, 1, 2, 3, 5]: + raise NewAttributeError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 5 or None'.format(distribution)) if distribution is not None: to_return['distribution'] = distribution @@ -356,9 +354,9 @@ class PyMISP(object): response = self.update_event(event['Event']['id'], event, 'json') return self._check_response(response) - def add_tag(self,event, tag): + def add_tag(self, event, tag): session = self.__prepare_session('json') - to_post = {'request': {'Event':{'id': event['Event']['id'], 'tag': tag}}} + to_post = {'request': {'Event': {'id': event['Event']['id'], 'tag': tag}}} response = session.post(urljoin(self.root_url, 'events/addTag'), data=json.dumps(to_post)) return self._check_response(response) @@ -847,8 +845,8 @@ class PyMISP(object): to_return.append(tag['name']) return to_return - def new_tag(self,name=None, colour="#00ace6", exportable=False): - to_post = {'Tag': {'name':name,'colour':colour, 'exportable':exportable}} + def new_tag(self, name=None, colour="#00ace6", exportable=False): + to_post = {'Tag': {'name': name, 'colour': colour, 'exportable': exportable}} session = self.__prepare_session('json') url = urljoin(self.root_url, 'tags/add') response = session.post(url, data=json.dumps(to_post)) @@ -897,10 +895,11 @@ class PyMISP(object): def get_all_attributes_txt(self, type_attr): session = self.__prepare_session('txt') - url = urljoin(self.root_url,'attributes/text/download/%s' % type_attr) + url = urljoin(self.root_url, 'attributes/text/download/%s' % type_attr) response = session.get(url) return response # ############## Deprecated (Pure XML API should not be used) ################## + @deprecated def download_all(self): """ From 6656e63dcc0ab5280b76364696eae9df85c2c412 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 21 Mar 2016 14:59:39 +0100 Subject: [PATCH 008/223] Update version to 2.3 --- pymisp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index d93ebf1..fd46fe5 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,3 +1,3 @@ -__version__ = '2.2' +__version__ = '2.3' from .api import PyMISP, PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey From dca755ef0845d5e9b76b3bada5faef56131390ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 21 Mar 2016 21:24:15 +0100 Subject: [PATCH 009/223] Improve examples --- examples/copy_list.py | 2 +- examples/create_events.py | 2 +- examples/get_network_activity.py | 2 +- examples/last.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) mode change 100644 => 100755 examples/copy_list.py diff --git a/examples/copy_list.py b/examples/copy_list.py old mode 100644 new mode 100755 index b7a1a55..bd60e5b --- a/examples/copy_list.py +++ b/examples/copy_list.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # -*- coding: utf-8 -*- import sys diff --git a/examples/create_events.py b/examples/create_events.py index 2f0e68a..d7e8649 100755 --- a/examples/create_events.py +++ b/examples/create_events.py @@ -16,7 +16,7 @@ def init(url, key): return PyMISP(url, key, True, 'json') if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Send malware sample to MISP.') + parser = argparse.ArgumentParser(description='Create an event on MISP.') parser.add_argument("-d", "--distrib", type=int, help="The distribution setting used for the attributes and for the newly created event, if relevant. [0-3].") parser.add_argument("-i", "--info", help="Used to populate the event info field if no event ID supplied.") parser.add_argument("-a", "--analysis", type=int, help="The analysis level of the newly created event, if applicatble. [0-2]") diff --git a/examples/get_network_activity.py b/examples/get_network_activity.py index e854393..433a9d8 100755 --- a/examples/get_network_activity.py +++ b/examples/get_network_activity.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # -*- coding: utf-8 -*- """ diff --git a/examples/last.py b/examples/last.py index 5eab820..75f8162 100755 --- a/examples/last.py +++ b/examples/last.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- from pymisp import PyMISP -from keys import misp_url, misp_key,misp_verifycert +from keys import misp_url, misp_key, misp_verifycert import argparse import os import json From 6db19ace9eb7d69aecd4708a000b13e9eba741aa Mon Sep 17 00:00:00 2001 From: Tristan METAYER Date: Wed, 23 Mar 2016 14:40:44 +0100 Subject: [PATCH 010/223] Add upload_attachment --- pymisp/api.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/pymisp/api.py b/pymisp/api.py index 1ea3b45..f12b668 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -566,7 +566,7 @@ class PyMISP(object): def prepare_attribute(self, event_id, distribution, to_ids, category, info, analysis, threat_level_id): to_post = {'request': {}} - authorized_categs = ['Payload delivery', 'Artifacts dropped', 'Payload Installation', 'External Analysis'] + authorized_categs = ['Payload delivery', 'Artifacts dropped', 'Payload Installation', 'External Analysis','Antivirus detection'] if event_id is not None: try: @@ -618,6 +618,32 @@ class PyMISP(object): response = session.post(url, data=json.dumps(to_post)) return self._check_response(response) + def upload_attachment(self, filename, filepath, event_id, distribution, to_ids, + category, info, analysis, threat_level_id): + to_post = self.prepare_attribute(event_id, distribution, to_ids, category, + info, analysis, threat_level_id) + to_post['request']['files'] = [{'filename': filename, 'data': self._encode_file_to_upload(filepath)}] + return self._upload_sample(to_post) + + def upload_attachmentlist(self, filepaths, event_id, distribution, to_ids, category, + info, analysis, threat_level_id): + to_post = self.prepare_attribute(event_id, distribution, to_ids, category, + info, analysis, threat_level_id) + files = [] + for path in filepaths: + if not os.path.isfile(path): + continue + files.append({'filename': os.path.basename(path), 'data': self._encode_file_to_upload(path)}) + to_post['request']['files'] = files + return self._upload_sample(to_post) + + def _upload_attachment(self, to_post): + session = self.__prepare_session('json') + url = urljoin(self.root_url, 'events/upload_attachment') + response = session.post(url, data=json.dumps(to_post)) + return self._check_response(response) + + # ############################ # ######## Proposals ######### # ############################ From f54e92ab81c9ac8782a5f14c8d3e6407394c7fa2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 24 Mar 2016 14:36:30 +0100 Subject: [PATCH 011/223] Fix pep8 --- pymisp/api.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index f12b668..0babf3c 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -566,7 +566,7 @@ class PyMISP(object): def prepare_attribute(self, event_id, distribution, to_ids, category, info, analysis, threat_level_id): to_post = {'request': {}} - authorized_categs = ['Payload delivery', 'Artifacts dropped', 'Payload Installation', 'External Analysis','Antivirus detection'] + authorized_categs = ['Payload delivery', 'Artifacts dropped', 'Payload Installation', 'External Analysis', 'Antivirus detection'] if event_id is not None: try: @@ -619,14 +619,14 @@ class PyMISP(object): return self._check_response(response) def upload_attachment(self, filename, filepath, event_id, distribution, to_ids, - category, info, analysis, threat_level_id): + category, info, analysis, threat_level_id): to_post = self.prepare_attribute(event_id, distribution, to_ids, category, info, analysis, threat_level_id) to_post['request']['files'] = [{'filename': filename, 'data': self._encode_file_to_upload(filepath)}] return self._upload_sample(to_post) def upload_attachmentlist(self, filepaths, event_id, distribution, to_ids, category, - info, analysis, threat_level_id): + info, analysis, threat_level_id): to_post = self.prepare_attribute(event_id, distribution, to_ids, category, info, analysis, threat_level_id) files = [] @@ -636,14 +636,13 @@ class PyMISP(object): files.append({'filename': os.path.basename(path), 'data': self._encode_file_to_upload(path)}) to_post['request']['files'] = files return self._upload_sample(to_post) - + def _upload_attachment(self, to_post): session = self.__prepare_session('json') url = urljoin(self.root_url, 'events/upload_attachment') response = session.post(url, data=json.dumps(to_post)) return self._check_response(response) - # ############################ # ######## Proposals ######### # ############################ From 2c134f61543ed51d6c390d388e46cc6c072d4b0e Mon Sep 17 00:00:00 2001 From: = <=> Date: Thu, 31 Mar 2016 12:33:04 +0100 Subject: [PATCH 012/223] Designed in same style as search, all attributes can be used --- pymisp/api.py | 46 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 70a9c81..316d14c 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -48,6 +48,8 @@ class NewEventError(PyMISPError): class NewAttributeError(PyMISPError): pass +class SearchError(PyMISPError): + pass class MissingDependency(PyMISPError): pass @@ -704,10 +706,48 @@ class PyMISP(object): response = session.post(url, data=json.dumps(query)) return self._check_response(response) - def search_index(self, value): - value = quote(value) + def search_index(self, published=None, eventid = None, tag = None, datefrom = None, + dateto = None, eventinfo = None, threatlevel = None, distribution = None, + analysis = None, attribute = None, org=None): + """ + Search only at the index level. Use ! infront of value as NOT, default OR + + :param published: Published (0,1) + :param eventid: Evend ID(s) | str or list + :param tag: Tag(s) | str or list + :param datefrom: First date, in format YYYY-MM-DD + :param datefrom: Last date, in format YYYY-MM-DD + :param eventinfo: Event info(s) to match | str or list + :param threatlevel: Threat level(s) (1,2,3,4) | str or list + :param distribution: Distribution level(s) (0,1,2,3) | str or list + :param analysis: Analysis level(s) (0,1,2) | str or list + :param org: Organisation(s) | str or list + + """ + allowed = {'published':published, 'eventid':eventid, 'tag':tag, 'Dateto':dateto, + 'Datefrom':datefrom, 'eventinfo':eventinfo, 'threatlevel':threatlevel, + 'distribution':distribution, 'analysis':analysis, 'attribute':attribute, + 'org':org } + rule_levels = {'distribution':["0","1","2","3","!0","!1","!2","!3"], + 'threatlevel':["1","2","3","4","!1","!2","!3","!4"], + 'analysis':["0","1","2","!0","!1","!2"]} + buildup_url = "events/index" + + for rule in allowed.keys(): + if allowed[rule] != None: + if type(allowed[rule])!=list: + allowed[rule]=[allowed[rule]] + allowed[rule] = map(str, allowed[rule]) + if rule in rule_levels: + if not set(allowed[rule]).issubset(rule_levels[rule]): + raise SearchError('Values in your {} are invalid, has to be in {}'.format(rule, ', '.join(str(x) for x in rule_levels[rule]))) + if type(allowed[rule]) == list: + joined = '|'.join(str(x) for x in allowed[rule]) + buildup_url += '/search{}:{}'.format(rule, joined) + else: + buildup_url += '/search{}:{}'.format(rule, allowed[rule]) session = self.__prepare_session('json') - url = urljoin(self.root_url, 'events/index/searchall:%s' % value) + url = urljoin(self.root_url, buildup_url) response = session.get(url) return self._check_response(response) From 1b7877dd0652ff12a6fdef5b51d392a578f575e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 4 Apr 2016 18:26:05 +0200 Subject: [PATCH 013/223] Use correct function to upload an attachment Fix #33 --- pymisp/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 316d14c..7f835a8 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -629,7 +629,7 @@ class PyMISP(object): to_post = self.prepare_attribute(event_id, distribution, to_ids, category, info, analysis, threat_level_id) to_post['request']['files'] = [{'filename': filename, 'data': self._encode_file_to_upload(filepath)}] - return self._upload_sample(to_post) + return self._upload_attachment(to_post) def upload_attachmentlist(self, filepaths, event_id, distribution, to_ids, category, info, analysis, threat_level_id): @@ -641,7 +641,7 @@ class PyMISP(object): continue files.append({'filename': os.path.basename(path), 'data': self._encode_file_to_upload(path)}) to_post['request']['files'] = files - return self._upload_sample(to_post) + return self._upload_attachment(to_post) def _upload_attachment(self, to_post): session = self.__prepare_session('json') From cb09a19e24fb68c497d79867b3e5e55df9fa5f69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 4 Apr 2016 18:34:08 +0200 Subject: [PATCH 014/223] Make PEP8 happy --- pymisp/api.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 7f835a8..00c690a 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -11,9 +11,7 @@ import re try: from urllib.parse import urljoin - from urllib.parse import quote except ImportError: - from urllib import quote from urlparse import urljoin from io import BytesIO import zipfile @@ -48,9 +46,11 @@ class NewEventError(PyMISPError): class NewAttributeError(PyMISPError): pass + class SearchError(PyMISPError): pass + class MissingDependency(PyMISPError): pass @@ -174,8 +174,8 @@ class PyMISP(object): raise PyMISPError('Unknown error: {}'.format(response.text)) errors = [] - if type(to_return) is list: - to_return = {'response':to_return} + if isinstance(to_return, list): + to_return = {'response': to_return} if to_return.get('error'): if not isinstance(to_return['error'], list): errors.append(to_return['error']) @@ -706,9 +706,9 @@ class PyMISP(object): response = session.post(url, data=json.dumps(query)) return self._check_response(response) - def search_index(self, published=None, eventid = None, tag = None, datefrom = None, - dateto = None, eventinfo = None, threatlevel = None, distribution = None, - analysis = None, attribute = None, org=None): + def search_index(self, published=None, eventid=None, tag=None, datefrom=None, + dateto=None, eventinfo=None, threatlevel=None, distribution=None, + analysis=None, attribute=None, org=None): """ Search only at the index level. Use ! infront of value as NOT, default OR @@ -724,19 +724,19 @@ class PyMISP(object): :param org: Organisation(s) | str or list """ - allowed = {'published':published, 'eventid':eventid, 'tag':tag, 'Dateto':dateto, - 'Datefrom':datefrom, 'eventinfo':eventinfo, 'threatlevel':threatlevel, - 'distribution':distribution, 'analysis':analysis, 'attribute':attribute, - 'org':org } - rule_levels = {'distribution':["0","1","2","3","!0","!1","!2","!3"], - 'threatlevel':["1","2","3","4","!1","!2","!3","!4"], - 'analysis':["0","1","2","!0","!1","!2"]} + allowed = {'published': published, 'eventid': eventid, 'tag': tag, 'Dateto': dateto, + 'Datefrom': datefrom, 'eventinfo': eventinfo, 'threatlevel': threatlevel, + 'distribution': distribution, 'analysis': analysis, 'attribute': attribute, + 'org': org} + rule_levels = {'distribution': ["0", "1", "2", "3", "!0", "!1", "!2", "!3"], + 'threatlevel': ["1", "2", "3", "4", "!1", "!2", "!3", "!4"], + 'analysis': ["0", "1", "2", "!0", "!1", "!2"]} buildup_url = "events/index" for rule in allowed.keys(): - if allowed[rule] != None: - if type(allowed[rule])!=list: - allowed[rule]=[allowed[rule]] + if allowed[rule] is not None: + if not isinstance(allowed[rule], list): + allowed[rule] = [allowed[rule]] allowed[rule] = map(str, allowed[rule]) if rule in rule_levels: if not set(allowed[rule]).issubset(rule_levels[rule]): From 9920d7686d848d6353141bbef1c11a04d4846542 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 4 Apr 2016 18:45:45 +0200 Subject: [PATCH 015/223] Revert "Use correct function to upload an attachment" This reverts commit 1b7877dd0652ff12a6fdef5b51d392a578f575e7. --- pymisp/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 00c690a..1a5e19d 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -629,7 +629,7 @@ class PyMISP(object): to_post = self.prepare_attribute(event_id, distribution, to_ids, category, info, analysis, threat_level_id) to_post['request']['files'] = [{'filename': filename, 'data': self._encode_file_to_upload(filepath)}] - return self._upload_attachment(to_post) + return self._upload_sample(to_post) def upload_attachmentlist(self, filepaths, event_id, distribution, to_ids, category, info, analysis, threat_level_id): @@ -641,7 +641,7 @@ class PyMISP(object): continue files.append({'filename': os.path.basename(path), 'data': self._encode_file_to_upload(path)}) to_post['request']['files'] = files - return self._upload_attachment(to_post) + return self._upload_sample(to_post) def _upload_attachment(self, to_post): session = self.__prepare_session('json') From 14f05fc9af80f19e3027a23d1b79307473d5d4b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 4 Apr 2016 18:48:27 +0200 Subject: [PATCH 016/223] Revert "Add upload_attachment" This reverts commit 6db19ace9eb7d69aecd4708a000b13e9eba741aa. --- pymisp/api.py | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 1a5e19d..df7231c 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -572,7 +572,7 @@ class PyMISP(object): def prepare_attribute(self, event_id, distribution, to_ids, category, info, analysis, threat_level_id): to_post = {'request': {}} - authorized_categs = ['Payload delivery', 'Artifacts dropped', 'Payload Installation', 'External Analysis', 'Antivirus detection'] + authorized_categs = ['Payload delivery', 'Artifacts dropped', 'Payload Installation', 'External Analysis'] if event_id is not None: try: @@ -624,31 +624,6 @@ class PyMISP(object): response = session.post(url, data=json.dumps(to_post)) return self._check_response(response) - def upload_attachment(self, filename, filepath, event_id, distribution, to_ids, - category, info, analysis, threat_level_id): - to_post = self.prepare_attribute(event_id, distribution, to_ids, category, - info, analysis, threat_level_id) - to_post['request']['files'] = [{'filename': filename, 'data': self._encode_file_to_upload(filepath)}] - return self._upload_sample(to_post) - - def upload_attachmentlist(self, filepaths, event_id, distribution, to_ids, category, - info, analysis, threat_level_id): - to_post = self.prepare_attribute(event_id, distribution, to_ids, category, - info, analysis, threat_level_id) - files = [] - for path in filepaths: - if not os.path.isfile(path): - continue - files.append({'filename': os.path.basename(path), 'data': self._encode_file_to_upload(path)}) - to_post['request']['files'] = files - return self._upload_sample(to_post) - - def _upload_attachment(self, to_post): - session = self.__prepare_session('json') - url = urljoin(self.root_url, 'events/upload_attachment') - response = session.post(url, data=json.dumps(to_post)) - return self._check_response(response) - # ############################ # ######## Proposals ######### # ############################ From 7be215a7323b9cc3ab965641ef804ab82b4a90c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 5 Apr 2016 02:03:30 +0200 Subject: [PATCH 017/223] Ann missing categories in the authorized ones. --- pymisp/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/api.py b/pymisp/api.py index df7231c..9f455ee 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -572,7 +572,7 @@ class PyMISP(object): def prepare_attribute(self, event_id, distribution, to_ids, category, info, analysis, threat_level_id): to_post = {'request': {}} - authorized_categs = ['Payload delivery', 'Artifacts dropped', 'Payload Installation', 'External Analysis'] + authorized_categs = ['Payload delivery', 'Artifacts dropped', 'Payload Installation', 'External Analysis', 'Network activity', 'Antivirus detection'] if event_id is not None: try: From 7f00da078850fe89532f2aa9dfdac318aa009dc2 Mon Sep 17 00:00:00 2001 From: Iglocska Date: Wed, 6 Apr 2016 11:49:19 +0200 Subject: [PATCH 018/223] Capitalisation issues --- pymisp/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/api.py b/pymisp/api.py index 9f455ee..c04e77d 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -572,7 +572,7 @@ class PyMISP(object): def prepare_attribute(self, event_id, distribution, to_ids, category, info, analysis, threat_level_id): to_post = {'request': {}} - authorized_categs = ['Payload delivery', 'Artifacts dropped', 'Payload Installation', 'External Analysis', 'Network activity', 'Antivirus detection'] + authorized_categs = ['Payload delivery', 'Artifacts dropped', 'Payload installation', 'External analysis', 'Network activity', 'Antivirus detection'] if event_id is not None: try: From b573daf86d2bcb5d8dc71e45a65b5f2ffc0866b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Fri, 8 Apr 2016 10:06:35 +0200 Subject: [PATCH 019/223] Correct module help --- examples/create_events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/create_events.py b/examples/create_events.py index d7e8649..17672eb 100755 --- a/examples/create_events.py +++ b/examples/create_events.py @@ -20,7 +20,7 @@ if __name__ == '__main__': parser.add_argument("-d", "--distrib", type=int, help="The distribution setting used for the attributes and for the newly created event, if relevant. [0-3].") parser.add_argument("-i", "--info", help="Used to populate the event info field if no event ID supplied.") parser.add_argument("-a", "--analysis", type=int, help="The analysis level of the newly created event, if applicatble. [0-2]") - parser.add_argument("-t", "--threat", type=int, help="The threat level ID of the newly created event, if applicatble. [0-3]") + parser.add_argument("-t", "--threat", type=int, help="The threat level ID of the newly created event, if applicatble. [1-4]") args = parser.parse_args() misp = init(misp_url, misp_key) From 1d4261fa5a393d08d1ee7d362c0c1bc51270109c Mon Sep 17 00:00:00 2001 From: Iglocska Date: Mon, 11 Apr 2016 15:18:05 +0200 Subject: [PATCH 020/223] Added the option to filter out attributes based on distribution level --- examples/feed-generator/generate.py | 19 ++++++++++++++++++- .../{settings.py => settings.default.py} | 16 ++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) rename examples/feed-generator/{settings.py => settings.default.py} (58%) diff --git a/examples/feed-generator/generate.py b/examples/feed-generator/generate.py index 13229b9..361fed3 100755 --- a/examples/feed-generator/generate.py +++ b/examples/feed-generator/generate.py @@ -5,7 +5,7 @@ import sys import json import os from pymisp import PyMISP -from settings import url, key, ssl, outputdir, filters +from settings import * objectsToSave = { @@ -29,8 +29,16 @@ fieldsToSave = ['uuid', 'info', 'threat_level_id', 'analysis', 'timestamp', 'publish_timestamp', 'published', 'date'] +valid_attribute_distributions = [] + def init(): + # If we have an old settings.py file then this variable won't exist + global valid_attribute_distributions + try: + valid_attribute_distributions = valid_attribute_distribution_levels + except: + valid_attribute_distributions = ['0', '1', '2', '3', '4', '5'] return PyMISP(url, key, ssl, 'json') @@ -61,11 +69,20 @@ def __cleanupEventFields(event, temp): return event +def __blockAttributeByDistribution(attribute): + if attribute['distribution'] not in valid_attribute_distributions: + return True + return False + + def __cleanupEventObjects(event, temp): for objectType in objectsToSave.keys(): if objectsToSave[objectType]['multiple'] is True: if objectType in temp['Event']: for objectInstance in temp['Event'][objectType]: + if objectType is 'Attribute': + if __blockAttributeByDistribution(objectInstance): + continue tempObject = {} for field in objectsToSave[objectType]['fields']: if field in objectInstance.keys(): diff --git a/examples/feed-generator/settings.py b/examples/feed-generator/settings.default.py similarity index 58% rename from examples/feed-generator/settings.py rename to examples/feed-generator/settings.default.py index 7901a87..b80ba93 100755 --- a/examples/feed-generator/settings.py +++ b/examples/feed-generator/settings.default.py @@ -21,3 +21,19 @@ outputdir = 'output' # tlp:white and/or feed-export but exclude anything tagged privint filters = {} + +# By default all attributes will be included in the feed generation +# Remove the levels that you do not wish to include in the feed +# Use this to further narrow down what gets exported, for example: +# Setting this to ['3', '5'] will exclude any attributes from the feed that +# are not exportable to all or inherit the event +# +# The levels are as follows: +# 0: Your Organisation Only +# 1: This Community Only +# 2: Connected Communities +# 3: All +# 4: Sharing Group +# 5: Inherit Event +valid_attribute_distribution_levels = ['0', '1', '2', '3', '4', '5'] + From c523e8acf624f42780deb90e91d2538ecdbe49a1 Mon Sep 17 00:00:00 2001 From: Nick Driver Date: Tue, 12 Apr 2016 13:42:01 -0400 Subject: [PATCH 021/223] Add SSDEEP and FILENAME|SSDEEP support --- pymisp/api.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index c04e77d..d1e6094 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -109,8 +109,8 @@ class PyMISP(object): 'Payload delivery', 'Payload installation', 'Artifacts dropped', 'Persistence mechanism', 'Network activity', 'Payload type', 'Attribution', 'External analysis', 'Other'] - self.types = ['md5', 'sha1', 'sha256', 'filename', 'filename|md5', 'filename|sha1', - 'filename|sha256', 'ip-src', 'ip-dst', 'hostname', 'domain', 'url', + self.types = ['md5', 'sha1', 'sha256', 'ssdeep', 'filename', 'filename|md5', 'filename|sha1', + 'filename|sha256', 'filename|ssdeep', 'ip-src', 'ip-dst', 'hostname', 'domain', 'url', 'user-agent', 'http-method', 'regkey', 'regkey|value', 'AS', 'snort', 'pattern-in-file', 'pattern-in-traffic', 'pattern-in-memory', 'named pipe', 'mutex', 'vulnerability', 'attachment', 'malware-sample', 'link', 'comment', @@ -388,7 +388,7 @@ class PyMISP(object): response = self.update_event(event['Event']['id'], event, 'json') return self._check_response(response) - def add_hashes(self, event, category='Artifacts dropped', filename=None, md5=None, sha1=None, sha256=None, comment=None, to_ids=True, distribution=None, proposal=False): + def add_hashes(self, event, category='Artifacts dropped', filename=None, md5=None, sha1=None, sha256=None, ssdeep=None, comment=None, to_ids=True, distribution=None, proposal=False): categories = ['Payload delivery', 'Artifacts dropped', 'Payload installation', 'External analysis'] if category not in categories: raise NewAttributeError('{} is invalid, category has to be in {}'.format(category, (', '.join(categories)))) @@ -408,6 +408,10 @@ class PyMISP(object): if sha256: attributes.append(self._prepare_full_attribute(category, type_value.format('sha256'), value.format(sha256), to_ids, comment, distribution)) + if ssdeep: + attributes.append(self._prepare_full_attribute(category, type_value.format('ssdeep'), value.format(ssdeep), + to_ids, comment, distribution)) + return self._send_attributes(event, attributes, proposal) From c7b0b3df64deb8761ba18d4437fa7faf2ff4653e Mon Sep 17 00:00:00 2001 From: Nick Driver Date: Wed, 13 Apr 2016 15:40:31 -0400 Subject: [PATCH 022/223] Add internal reference attributes --- pymisp/api.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/pymisp/api.py b/pymisp/api.py index c04e77d..cc63f86 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -553,6 +553,28 @@ class PyMISP(object): attributes = [] attributes.append(self._prepare_full_attribute('Attribution', 'threat-actor', target, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) + + # ##### Internal reference attributes ##### + + def add_internal_link(self, event, reference, to_ids=False, comment=None, distribution=None, proposal=False): + attributes = [] + attributes.append(self._prepare_full_attribute('Internal reference', 'link', reference, to_ids, comment, distribution)) + return self._send_attributes(event, attributes, proposal) + + def add_internal_comment(self, event, reference, to_ids=False, comment=None, distribution=None, proposal=False): + attributes = [] + attributes.append(self._prepare_full_attribute('Internal reference', 'comment', reference, to_ids, comment, distribution)) + return self._send_attributes(event, attributes, proposal) + + def add_internal_text(self, event, reference, to_ids=False, comment=None, distribution=None, proposal=False): + attributes = [] + attributes.append(self._prepare_full_attribute('Internal reference', 'text', reference, to_ids, comment, distribution)) + return self._send_attributes(event, attributes, proposal) + + def add_internal_other(self, event, reference, to_ids=False, comment=None, distribution=None, proposal=False): + attributes = [] + attributes.append(self._prepare_full_attribute('Internal reference', 'other', reference, to_ids, comment, distribution)) + return self._send_attributes(event, attributes, proposal) # ################################################## # ######### Upload samples through the API ######### From 423757530bf42baaf362119a294b403cf3259c1a Mon Sep 17 00:00:00 2001 From: Iglocska Date: Mon, 11 Apr 2016 15:18:05 +0200 Subject: [PATCH 023/223] Added the option to filter out attributes based on distribution level --- examples/feed-generator/generate.py | 19 ++++++++++++++++++- .../{settings.py => settings.default.py} | 16 ++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) rename examples/feed-generator/{settings.py => settings.default.py} (58%) diff --git a/examples/feed-generator/generate.py b/examples/feed-generator/generate.py index 13229b9..361fed3 100755 --- a/examples/feed-generator/generate.py +++ b/examples/feed-generator/generate.py @@ -5,7 +5,7 @@ import sys import json import os from pymisp import PyMISP -from settings import url, key, ssl, outputdir, filters +from settings import * objectsToSave = { @@ -29,8 +29,16 @@ fieldsToSave = ['uuid', 'info', 'threat_level_id', 'analysis', 'timestamp', 'publish_timestamp', 'published', 'date'] +valid_attribute_distributions = [] + def init(): + # If we have an old settings.py file then this variable won't exist + global valid_attribute_distributions + try: + valid_attribute_distributions = valid_attribute_distribution_levels + except: + valid_attribute_distributions = ['0', '1', '2', '3', '4', '5'] return PyMISP(url, key, ssl, 'json') @@ -61,11 +69,20 @@ def __cleanupEventFields(event, temp): return event +def __blockAttributeByDistribution(attribute): + if attribute['distribution'] not in valid_attribute_distributions: + return True + return False + + def __cleanupEventObjects(event, temp): for objectType in objectsToSave.keys(): if objectsToSave[objectType]['multiple'] is True: if objectType in temp['Event']: for objectInstance in temp['Event'][objectType]: + if objectType is 'Attribute': + if __blockAttributeByDistribution(objectInstance): + continue tempObject = {} for field in objectsToSave[objectType]['fields']: if field in objectInstance.keys(): diff --git a/examples/feed-generator/settings.py b/examples/feed-generator/settings.default.py similarity index 58% rename from examples/feed-generator/settings.py rename to examples/feed-generator/settings.default.py index 7901a87..b80ba93 100755 --- a/examples/feed-generator/settings.py +++ b/examples/feed-generator/settings.default.py @@ -21,3 +21,19 @@ outputdir = 'output' # tlp:white and/or feed-export but exclude anything tagged privint filters = {} + +# By default all attributes will be included in the feed generation +# Remove the levels that you do not wish to include in the feed +# Use this to further narrow down what gets exported, for example: +# Setting this to ['3', '5'] will exclude any attributes from the feed that +# are not exportable to all or inherit the event +# +# The levels are as follows: +# 0: Your Organisation Only +# 1: This Community Only +# 2: Connected Communities +# 3: All +# 4: Sharing Group +# 5: Inherit Event +valid_attribute_distribution_levels = ['0', '1', '2', '3', '4', '5'] + From 887a2b49b193004da6aea9e4ec62bfeee2e424bb Mon Sep 17 00:00:00 2001 From: Nick Driver Date: Wed, 13 Apr 2016 15:40:31 -0400 Subject: [PATCH 024/223] Add internal reference attributes --- pymisp/api.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/pymisp/api.py b/pymisp/api.py index c04e77d..cc63f86 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -553,6 +553,28 @@ class PyMISP(object): attributes = [] attributes.append(self._prepare_full_attribute('Attribution', 'threat-actor', target, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) + + # ##### Internal reference attributes ##### + + def add_internal_link(self, event, reference, to_ids=False, comment=None, distribution=None, proposal=False): + attributes = [] + attributes.append(self._prepare_full_attribute('Internal reference', 'link', reference, to_ids, comment, distribution)) + return self._send_attributes(event, attributes, proposal) + + def add_internal_comment(self, event, reference, to_ids=False, comment=None, distribution=None, proposal=False): + attributes = [] + attributes.append(self._prepare_full_attribute('Internal reference', 'comment', reference, to_ids, comment, distribution)) + return self._send_attributes(event, attributes, proposal) + + def add_internal_text(self, event, reference, to_ids=False, comment=None, distribution=None, proposal=False): + attributes = [] + attributes.append(self._prepare_full_attribute('Internal reference', 'text', reference, to_ids, comment, distribution)) + return self._send_attributes(event, attributes, proposal) + + def add_internal_other(self, event, reference, to_ids=False, comment=None, distribution=None, proposal=False): + attributes = [] + attributes.append(self._prepare_full_attribute('Internal reference', 'other', reference, to_ids, comment, distribution)) + return self._send_attributes(event, attributes, proposal) # ################################################## # ######### Upload samples through the API ######### From 9e92072f884492b636cd89c57c982c0000231bde Mon Sep 17 00:00:00 2001 From: Nick Driver Date: Tue, 12 Apr 2016 13:42:01 -0400 Subject: [PATCH 025/223] Add SSDEEP and FILENAME|SSDEEP support --- pymisp/api.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index cc63f86..f261222 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -109,8 +109,8 @@ class PyMISP(object): 'Payload delivery', 'Payload installation', 'Artifacts dropped', 'Persistence mechanism', 'Network activity', 'Payload type', 'Attribution', 'External analysis', 'Other'] - self.types = ['md5', 'sha1', 'sha256', 'filename', 'filename|md5', 'filename|sha1', - 'filename|sha256', 'ip-src', 'ip-dst', 'hostname', 'domain', 'url', + self.types = ['md5', 'sha1', 'sha256', 'ssdeep', 'filename', 'filename|md5', 'filename|sha1', + 'filename|sha256', 'filename|ssdeep', 'ip-src', 'ip-dst', 'hostname', 'domain', 'url', 'user-agent', 'http-method', 'regkey', 'regkey|value', 'AS', 'snort', 'pattern-in-file', 'pattern-in-traffic', 'pattern-in-memory', 'named pipe', 'mutex', 'vulnerability', 'attachment', 'malware-sample', 'link', 'comment', @@ -388,7 +388,7 @@ class PyMISP(object): response = self.update_event(event['Event']['id'], event, 'json') return self._check_response(response) - def add_hashes(self, event, category='Artifacts dropped', filename=None, md5=None, sha1=None, sha256=None, comment=None, to_ids=True, distribution=None, proposal=False): + def add_hashes(self, event, category='Artifacts dropped', filename=None, md5=None, sha1=None, sha256=None, ssdeep=None, comment=None, to_ids=True, distribution=None, proposal=False): categories = ['Payload delivery', 'Artifacts dropped', 'Payload installation', 'External analysis'] if category not in categories: raise NewAttributeError('{} is invalid, category has to be in {}'.format(category, (', '.join(categories)))) @@ -408,6 +408,10 @@ class PyMISP(object): if sha256: attributes.append(self._prepare_full_attribute(category, type_value.format('sha256'), value.format(sha256), to_ids, comment, distribution)) + if ssdeep: + attributes.append(self._prepare_full_attribute(category, type_value.format('ssdeep'), value.format(ssdeep), + to_ids, comment, distribution)) + return self._send_attributes(event, attributes, proposal) From d493ff76baa5c6f4fe37fda5be2ac859ea4aee90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Thu, 14 Apr 2016 10:29:36 +0200 Subject: [PATCH 026/223] type-category association checking automated --- pymisp/api.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index f261222..c221028 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -105,6 +105,19 @@ class PyMISP(object): self.out_type = out_type self.debug = debug + try: + # Make sure the MISP instance is working and the URL is valid + self.get_version() + except Exception as e: + raise PyMISPError('Unable to connect to MISP ({}). Please make sure the API key and the URL are correct (http/https is required): {}'.format(self.root_url, e)) + + session = self.__prepare_session(out_type) + self.describe_types = session.get(self.root_url + 'attributes/describeTypes.json').json() + + self.categories = self.describe_types['result']['categories'] + self.types = self.describe_types['result']['types'] + self.category_type_mapping = self.describe_types['result']['category_type_mappings'] + self.categories = ['Internal reference', 'Targeting data', 'Antivirus detection', 'Payload delivery', 'Payload installation', 'Artifacts dropped', 'Persistence mechanism', 'Network activity', 'Payload type', @@ -118,11 +131,7 @@ class PyMISP(object): 'yara', 'target-user', 'target-email', 'target-machine', 'target-org', 'target-location', 'target-external', 'other', 'threat-actor'] - try: - # Make sure the MISP instance is working and the URL is valid - self.get_version() - except Exception as e: - raise PyMISPError('Unable to connect to MISP ({}). Please make sure the API key and the URL are correct (http/https is required): {}'.format(self.root_url, e)) + def __prepare_session(self, force_out=None): """ @@ -296,11 +305,14 @@ class PyMISP(object): to_return = {} if category not in self.categories: raise NewAttributeError('{} is invalid, category has to be in {}'.format(category, (', '.join(self.categories)))) - to_return['category'] = category if type_value not in self.types: raise NewAttributeError('{} is invalid, type_value has to be in {}'.format(type_value, (', '.join(self.types)))) + + if type_value not in self.category_type_mapping[category]: + raise NewAttributeError('{} and {} is an invalid combinaison, type_value for this category has to be in {}'.format(type_value, category, (', '.join(self.category_type_mapping[category])))) to_return['type'] = type_value + to_return['category'] = category if to_ids not in [True, False]: raise NewAttributeError('{} is invalid, to_ids has to be True or False'.format(to_ids)) From c269913ad3ba2b03c898ff92b17ff3a6301a5f81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Thu, 14 Apr 2016 10:47:13 +0200 Subject: [PATCH 027/223] type-category association checking automated --- pymisp/api.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index f261222..94a9bb3 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -105,25 +105,19 @@ class PyMISP(object): self.out_type = out_type self.debug = debug - self.categories = ['Internal reference', 'Targeting data', 'Antivirus detection', - 'Payload delivery', 'Payload installation', 'Artifacts dropped', - 'Persistence mechanism', 'Network activity', 'Payload type', - 'Attribution', 'External analysis', 'Other'] - self.types = ['md5', 'sha1', 'sha256', 'ssdeep', 'filename', 'filename|md5', 'filename|sha1', - 'filename|sha256', 'filename|ssdeep', 'ip-src', 'ip-dst', 'hostname', 'domain', 'url', - 'user-agent', 'http-method', 'regkey', 'regkey|value', 'AS', 'snort', - 'pattern-in-file', 'pattern-in-traffic', 'pattern-in-memory', 'named pipe', - 'mutex', 'vulnerability', 'attachment', 'malware-sample', 'link', 'comment', - 'text', 'email-src', 'email-dst', 'email-subject', 'email-attachment', - 'yara', 'target-user', 'target-email', 'target-machine', 'target-org', - 'target-location', 'target-external', 'other', 'threat-actor'] - try: # Make sure the MISP instance is working and the URL is valid self.get_version() except Exception as e: raise PyMISPError('Unable to connect to MISP ({}). Please make sure the API key and the URL are correct (http/https is required): {}'.format(self.root_url, e)) + session = self.__prepare_session(out_type) + self.describe_types = session.get(self.root_url + 'attributes/describeTypes.json').json() + + self.categories = self.describe_types['result']['categories'] + self.types = self.describe_types['result']['types'] + self.category_type_mapping = self.describe_types['result']['category_type_mappings'] + def __prepare_session(self, force_out=None): """ Prepare the headers of the session @@ -296,11 +290,14 @@ class PyMISP(object): to_return = {} if category not in self.categories: raise NewAttributeError('{} is invalid, category has to be in {}'.format(category, (', '.join(self.categories)))) - to_return['category'] = category if type_value not in self.types: raise NewAttributeError('{} is invalid, type_value has to be in {}'.format(type_value, (', '.join(self.types)))) + + if type_value not in self.category_type_mapping[category]: + raise NewAttributeError('{} and {} is an invalid combinaison, type_value for this category has to be in {}'.format(type_value, category, (', '.join(self.category_type_mapping[category])))) to_return['type'] = type_value + to_return['category'] = category if to_ids not in [True, False]: raise NewAttributeError('{} is invalid, to_ids has to be True or False'.format(to_ids)) From 25bc301789e14889ca5c36e1168590c33e118769 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Thu, 14 Apr 2016 14:19:08 +0200 Subject: [PATCH 028/223] removing some unnecessary checks --- pymisp/api.py | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 4897d50..01cfdab 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -118,27 +118,6 @@ class PyMISP(object): self.types = self.describe_types['result']['types'] self.category_type_mapping = self.describe_types['result']['category_type_mappings'] - self.categories = ['Internal reference', 'Targeting data', 'Antivirus detection', - 'Payload delivery', 'Payload installation', 'Artifacts dropped', - 'Persistence mechanism', 'Network activity', 'Payload type', - 'Attribution', 'External analysis', 'Other'] - self.types = ['md5', 'sha1', 'sha256', 'ssdeep', 'filename', 'filename|md5', 'filename|sha1', - 'filename|sha256', 'filename|ssdeep', 'ip-src', 'ip-dst', 'hostname', 'domain', 'url', - 'user-agent', 'http-method', 'regkey', 'regkey|value', 'AS', 'snort', - 'pattern-in-file', 'pattern-in-traffic', 'pattern-in-memory', 'named pipe', - 'mutex', 'vulnerability', 'attachment', 'malware-sample', 'link', 'comment', - 'text', 'email-src', 'email-dst', 'email-subject', 'email-attachment', - 'yara', 'target-user', 'target-email', 'target-machine', 'target-org', - 'target-location', 'target-external', 'other', 'threat-actor'] - - - session = self.__prepare_session(out_type) - self.describe_types = session.get(self.root_url + 'attributes/describeTypes.json').json() - - self.categories = self.describe_types['result']['categories'] - self.types = self.describe_types['result']['types'] - self.category_type_mapping = self.describe_types['result']['category_type_mappings'] - def __prepare_session(self, force_out=None): """ Prepare the headers of the session @@ -407,9 +386,6 @@ class PyMISP(object): return self._check_response(response) def add_hashes(self, event, category='Artifacts dropped', filename=None, md5=None, sha1=None, sha256=None, ssdeep=None, comment=None, to_ids=True, distribution=None, proposal=False): - categories = ['Payload delivery', 'Artifacts dropped', 'Payload installation', 'External analysis'] - if category not in categories: - raise NewAttributeError('{} is invalid, category has to be in {}'.format(category, (', '.join(categories)))) attributes = [] type_value = '{}' @@ -520,9 +496,6 @@ class PyMISP(object): return self._send_attributes(event, attributes, proposal) def add_email_dst(self, event, email, category='Payload delivery', to_ids=True, comment=None, distribution=None, proposal=False): - categories = ['Payload delivery', 'Network activity'] - if category not in categories: - raise NewAttributeError('{} is invalid, category has to be in {}'.format(category, (', '.join(categories)))) attributes = [] attributes.append(self._prepare_full_attribute(category, 'email-dst', email, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) From 369922436ed2a546929f6ffa7be1d87611784405 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Thu, 14 Apr 2016 10:29:36 +0200 Subject: [PATCH 029/223] type-category association checking automated --- pymisp/api.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 01cfdab..73a16c7 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -406,7 +406,6 @@ class PyMISP(object): attributes.append(self._prepare_full_attribute(category, type_value.format('ssdeep'), value.format(ssdeep), to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) def add_regkey(self, event, regkey, rvalue=None, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False): @@ -548,24 +547,24 @@ class PyMISP(object): attributes = [] attributes.append(self._prepare_full_attribute('Attribution', 'threat-actor', target, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) - + # ##### Internal reference attributes ##### - + def add_internal_link(self, event, reference, to_ids=False, comment=None, distribution=None, proposal=False): attributes = [] attributes.append(self._prepare_full_attribute('Internal reference', 'link', reference, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) - + def add_internal_comment(self, event, reference, to_ids=False, comment=None, distribution=None, proposal=False): attributes = [] attributes.append(self._prepare_full_attribute('Internal reference', 'comment', reference, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) - + def add_internal_text(self, event, reference, to_ids=False, comment=None, distribution=None, proposal=False): attributes = [] attributes.append(self._prepare_full_attribute('Internal reference', 'text', reference, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) - + def add_internal_other(self, event, reference, to_ids=False, comment=None, distribution=None, proposal=False): attributes = [] attributes.append(self._prepare_full_attribute('Internal reference', 'other', reference, to_ids, comment, distribution)) From b94423781ca4313e9f6bfa504fa2b09985db0f0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Fri, 15 Apr 2016 13:47:13 +0200 Subject: [PATCH 030/223] add function add filename --- pymisp/api.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pymisp/api.py b/pymisp/api.py index 73a16c7..8ab9b72 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -408,6 +408,11 @@ class PyMISP(object): return self._send_attributes(event, attributes, proposal) + def add_filename(self, event, filename, category='Artifacts dropped', to_ids=False, comment=None, distribution=None, proposal=False): + attributes = [] + attributes.append(self._prepare_full_attribute(category, 'filename', filename, to_ids, comment, distribution)) + return self._send_attributes(event, attributes, proposal) + def add_regkey(self, event, regkey, rvalue=None, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False): type_value = '{}' value = '{}' From 089b0a72b02887611b07a330d3751ecb2c563f3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Fri, 15 Apr 2016 15:52:50 +0200 Subject: [PATCH 031/223] add comment field in upload_sample --- pymisp/api.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 8ab9b72..1faab13 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -583,14 +583,14 @@ class PyMISP(object): # Setup details of a new event if distribution not in [0, 1, 2, 3]: raise NewEventError('{} is invalid, the distribution has to be in 0, 1, 2, 3'.format(distribution)) - if threat_level_id not in [0, 1, 2, 3]: - raise NewEventError('{} is invalid, the threat_level_id has to be in 0, 1, 2, 3'.format(threat_level_id)) + if threat_level_id not in [1, 2, 3, 4]: + raise NewEventError('{} is invalid, the threat_level_id has to be in 1, 2, 3, 4'.format(threat_level_id)) if analysis not in [0, 1, 2]: raise NewEventError('{} is invalid, the analysis has to be in 0, 1, 2'.format(analysis)) return {'distribution': int(distribution), 'info': info, 'threat_level_id': int(threat_level_id), 'analysis': analysis} - def prepare_attribute(self, event_id, distribution, to_ids, category, info, + def prepare_attribute(self, event_id, distribution, to_ids, category, comment, info, analysis, threat_level_id): to_post = {'request': {}} authorized_categs = ['Payload delivery', 'Artifacts dropped', 'Payload installation', 'External analysis', 'Network activity', 'Antivirus detection'] @@ -614,6 +614,7 @@ class PyMISP(object): raise NewAttributeError('{} is invalid, category has to be in {}'.format(category, (', '.join(authorized_categs)))) to_post['request']['category'] = category + to_post['request']['comment'] = comment return to_post def _encode_file_to_upload(self, path): @@ -621,9 +622,9 @@ class PyMISP(object): return base64.b64encode(f.read()) def upload_sample(self, filename, filepath, event_id, distribution, to_ids, - category, info, analysis, threat_level_id): + category, comment, info, analysis, threat_level_id): to_post = self.prepare_attribute(event_id, distribution, to_ids, category, - info, analysis, threat_level_id) + comment, info, analysis, threat_level_id) to_post['request']['files'] = [{'filename': filename, 'data': self._encode_file_to_upload(filepath)}] return self._upload_sample(to_post) From 46396202f54ffc6287423ea9942ab20e7ab24945 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 15 Apr 2016 16:29:47 +0200 Subject: [PATCH 032/223] Update version to v2.4.36 --- pymisp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index fd46fe5..18eac4f 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,3 +1,3 @@ -__version__ = '2.3' +__version__ = '2.4.36' from .api import PyMISP, PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey From ea952a95886679f917f8ca20aec75ad56f6537d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 20 Apr 2016 10:16:44 +0200 Subject: [PATCH 033/223] Add 'add_yara' to upload yara rules, increase flexibility of config fix #38 --- pymisp/api.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pymisp/api.py b/pymisp/api.py index 1faab13..11e16dd 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -112,7 +112,7 @@ class PyMISP(object): raise PyMISPError('Unable to connect to MISP ({}). Please make sure the API key and the URL are correct (http/https is required): {}'.format(self.root_url, e)) session = self.__prepare_session(out_type) - self.describe_types = session.get(self.root_url + 'attributes/describeTypes.json').json() + self.describe_types = session.get(urljoin(self.root_url, 'attributes/describeTypes.json')).json() self.categories = self.describe_types['result']['categories'] self.types = self.describe_types['result']['types'] @@ -450,6 +450,11 @@ class PyMISP(object): attributes.append(self._prepare_full_attribute(category, 'mutex', mutex, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) + def add_yara(self, event, yara, category='Payload delivery', to_ids=False, comment=None, distribution=None, proposal=False): + attributes = [] + attributes.append(self._prepare_full_attribute(category, 'yara', yara, to_ids, comment, distribution)) + return self._send_attributes(event, attributes, proposal) + # ##### Network attributes ##### def add_ipdst(self, event, ipdst, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): From 5c23d12f2c91c85a955c67493ddedc09afb5b452 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Thu, 28 Apr 2016 13:29:54 +0200 Subject: [PATCH 034/223] add function get_attributes_statistics --- pymisp/api.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/pymisp/api.py b/pymisp/api.py index 11e16dd..7bf9f61 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -976,8 +976,24 @@ class PyMISP(object): url = urljoin(self.root_url, 'attributes/text/download/%s' % type_attr) response = session.get(url) return response - # ############## Deprecated (Pure XML API should not be used) ################## + # ############## Statistics ################## + + def get_attributes_statistics(self, context='type', percentage=None, force_out=None): + """ + Get statistics from the MISP instance + """ + session = self.__prepare_session(force_out) + if (context != 'category'): + context = 'type' + if(percentage!=None): + url = urljoin(self.root_url, 'attributes/attributeStatistics/{}/{}'.format(context, percentage)) + else: + url = urljoin(self.root_url, 'attributes/attributeStatistics/{}'.format(context)) + print(url) + return session.get(url).json() + + # ############## Deprecated (Pure XML API should not be used) ################## @deprecated def download_all(self): """ @@ -1001,3 +1017,5 @@ class PyMISP(object): template = urljoin(self.root_url, 'events/xml/download/{}/{}'.format(event_id, attach)) session = self.__prepare_session('xml') return session.get(template) + + From f490898a5c367ade69f60cd4653ce33338e174e8 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Thu, 28 Apr 2016 14:44:01 +0200 Subject: [PATCH 035/223] Statistics test script added --- examples/stats.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100755 examples/stats.py diff --git a/examples/stats.py b/examples/stats.py new file mode 100755 index 0000000..3ce789e --- /dev/null +++ b/examples/stats.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pymisp import PyMISP +from keys import misp_url, misp_key, misp_verifycert +import argparse + + +def init(url, key): + return PyMISP(url, key, misp_verifycert, 'json') + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Output attributes statistics from a MISP instance.') + args = parser.parse_args() + + misp = init(misp_url, misp_key) + + print misp.get_attributes_statistics(misp, percentage=True) From ca382960efaafed1cb7f4adebbbe72b0f7f4633a Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Thu, 28 Apr 2016 14:45:02 +0200 Subject: [PATCH 036/223] Debug print removed --- pymisp/api.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 7bf9f61..ba3df58 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -985,12 +985,11 @@ class PyMISP(object): """ session = self.__prepare_session(force_out) if (context != 'category'): - context = 'type' + context = 'type' if(percentage!=None): url = urljoin(self.root_url, 'attributes/attributeStatistics/{}/{}'.format(context, percentage)) else: url = urljoin(self.root_url, 'attributes/attributeStatistics/{}'.format(context)) - print(url) return session.get(url).json() # ############## Deprecated (Pure XML API should not be used) ################## From 7dc9e209973afc82b02d912ec94c7246d0307125 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Thu, 28 Apr 2016 15:05:31 +0200 Subject: [PATCH 037/223] More stats example --- examples/stats.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/stats.py b/examples/stats.py index 3ce789e..41d6b28 100755 --- a/examples/stats.py +++ b/examples/stats.py @@ -15,4 +15,5 @@ if __name__ == '__main__': misp = init(misp_url, misp_key) - print misp.get_attributes_statistics(misp, percentage=True) + print (misp.get_attributes_statistics(misp, percentage=True)) + print (misp.get_attributes_statistics(context='category', percentage=True)) From 3cd9ede99f7b884b227edb53fdc4f360b6c15692 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Fri, 29 Apr 2016 16:35:27 +0200 Subject: [PATCH 038/223] Add function for sighting using attribute id, uuid or a json file --- examples/sighting.json | 2 ++ examples/sighting.py | 27 +++++++++++++++++++++++++++ pymisp/api.py | 19 ++++++++++++++++++- 3 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 examples/sighting.json create mode 100644 examples/sighting.py diff --git a/examples/sighting.json b/examples/sighting.json new file mode 100644 index 0000000..9191a5b --- /dev/null +++ b/examples/sighting.json @@ -0,0 +1,2 @@ +{"values":["www.google.com", "8.8.8.8"], "timestamp":1460558710} + diff --git a/examples/sighting.py b/examples/sighting.py new file mode 100644 index 0000000..433c2c2 --- /dev/null +++ b/examples/sighting.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pymisp import PyMISP +from keys import misp_url, misp_key +import argparse + +# For python2 & 3 compat, a bit dirty, but it seems to be the least bad one +try: + input = raw_input +except NameError: + pass + + +def init(url, key): + return PyMISP(url, key, True, 'json') + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Add sighting.') + parser.add_argument("-f", "--json_file", help="The name of the json file describing the attribute you want to add sighting to.") + args = parser.parse_args() + + misp = init(misp_url, misp_key) + + misp.sighting_per_json(args.json_file) + + diff --git a/pymisp/api.py b/pymisp/api.py index 7bf9f61..9022d34 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -990,9 +990,26 @@ class PyMISP(object): url = urljoin(self.root_url, 'attributes/attributeStatistics/{}/{}'.format(context, percentage)) else: url = urljoin(self.root_url, 'attributes/attributeStatistics/{}'.format(context)) - print(url) return session.get(url).json() + # ############## Sightings ################## + + def sighting_per_id(self, attribute_id, force_out=None): + session = self.__prepare_session(force_out) + url = urljoin(self.root_url, 'sightings/add/{}'.format(attribute_id)) + return session.post(url) + + def sighting_per_uuid(self, attribute_uuid, force_out=None): + session = self.__prepare_session(force_out) + url = urljoin(self.root_url, 'sightings/add/{}'.format(attribute_uuid)) + return session.post(url) + + def sighting_per_json(self, json_file, force_out=None): + session = self.__prepare_session(force_out) + jdata = json.load(open(json_file)) + url = urljoin(self.root_url, 'sightings/add/') + return session.post(url, data=json.dumps(jdata)) + # ############## Deprecated (Pure XML API should not be used) ################## @deprecated def download_all(self): From 12fa199202ccd6ed72438b9bf314dece57c6627e Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Thu, 5 May 2016 10:05:59 +0100 Subject: [PATCH 039/223] Add Attribute by named category and type --- pymisp/api.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pymisp/api.py b/pymisp/api.py index a3a355c..8f8abeb 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -385,6 +385,15 @@ class PyMISP(object): response = self.update_event(event['Event']['id'], event, 'json') return self._check_response(response) + def add_named_attribute(self, event, category, type_value, value, to_ids=False, comment=None, distribution=None, proposal=False): + attributes = [] + if value and category and type: + try: + attributes.append(self._prepare_full_attribute(category, type_value, value, to_ids, comment, distribution)) + except NewAttributeError as e: + return e + return self._send_attributes(event, attributes, proposal) + def add_hashes(self, event, category='Artifacts dropped', filename=None, md5=None, sha1=None, sha256=None, ssdeep=None, comment=None, to_ids=True, distribution=None, proposal=False): attributes = [] From acf888165819f1f841777cbe571239ee53bdbf23 Mon Sep 17 00:00:00 2001 From: Tristan METAYER Date: Thu, 12 May 2016 17:33:13 +0200 Subject: [PATCH 040/223] init for ioc-2-misp --- examples/ioc-2-misp/README.md | 25 +++ examples/ioc-2-misp/ioc2misp.py | 348 +++++++++++++++++++++++++++++ examples/ioc-2-misp/keys.py.sample | 94 ++++++++ examples/ioc-2-misp/taxonomy.csv | 12 + 4 files changed, 479 insertions(+) create mode 100644 examples/ioc-2-misp/README.md create mode 100644 examples/ioc-2-misp/ioc2misp.py create mode 100644 examples/ioc-2-misp/keys.py.sample create mode 100644 examples/ioc-2-misp/taxonomy.csv diff --git a/examples/ioc-2-misp/README.md b/examples/ioc-2-misp/README.md new file mode 100644 index 0000000..dc22d58 --- /dev/null +++ b/examples/ioc-2-misp/README.md @@ -0,0 +1,25 @@ +### Description + +Python script for ioc import to misp + +### requires + +> python 2.7 +> PyMISP +> BeautifulSoup (apt-get install python-bs4 lxml) + +### Usage + +```bash +python ioc2misp.py -i myioc -t "tag:mytag='sample','tag:other='foo'" +``` + +```bash +time find /iocsample -type f|while read line ;do python ioc2misp.py -i ${line};done +``` + +### Conf + + * rename keys.py.sample as keys.py + * add your url and api key in keys.py + * use command in terminal \ No newline at end of file diff --git a/examples/ioc-2-misp/ioc2misp.py b/examples/ioc-2-misp/ioc2misp.py new file mode 100644 index 0000000..ecbeb60 --- /dev/null +++ b/examples/ioc-2-misp/ioc2misp.py @@ -0,0 +1,348 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Format description +# @variables : camelCase +# @functions : snake_case + +from keys import mispUrl, mispKey, csvTaxonomyFile, iocMispMapping + +try: + from pymisp import PyMISP +except: + print "you need pymisp form github" + import sys + sys.exit(1) + +import json +import os +import argparse + +try: + from bs4 import BeautifulSoup +except: + print "install BeautifulSoup : sudo apt-get install python-bs4 python-lxml" + import sys + sys.exit(1) + +def misp_init(url, key): + return PyMISP(url, key, False, 'json') + +def check_valid_ioc(): + + (filepath, filename) = os.path.split(iocDescriptions["iocfile"]) + (shortname, extension) = os.path.splitext(filename) + + if (("ioc" in extension)) and (sum(1 for _ in open(iocDescriptions["iocfile"])) > 1): + iocDescriptions['filename'] = filename + return True + return False + +def get_parse_ioc_file(): + + return BeautifulSoup(open(iocDescriptions["iocfile"]) , "lxml") + +def parse_ioc_search_content(iocContextSearch): + for k,v in iocMispMapping.items(): + if str(k).lower() == str(iocContextSearch).lower(): + return v + return False + +def create_attribute_json(iocContextSearch, attributeValue, attributeComment,force=False): + ##################################### + # force used for description to upload + if force: + parseResult=("Other","comment") + else: + parseResult = parse_ioc_search_content(iocContextSearch) + + if parseResult is False: + + print "/!\ Not implemented :: {0} :: {1} :: Item add as 'Other','Comment'. Add it in your keys.py".format(iocContextSearch,attributeValue) + ######################################## + # force import to misp + parseResult=("Other","comment") + + comment = "" + try : + comment= parseResult[2]+attributeComment + except: + comment= attributeComment + + attribute = { + "category": parseResult[0], + "type": parseResult[1], + "value": attributeValue, + "timestamp": "0", + "to_ids": "0", + "distribution": "0", + "comment": comment, + } + return attribute + +def create_attributes_from_ioc_json(soup): + attributes = [] + + IndicatorItemValues = {} + for item in soup.find_all("indicatoritem"): + + if item.find('context'): + IndicatorItemValues["context"]=str(item.find('context')['search']) + else: + IndicatorItemValues["context"]="" + if item.find('content'): + IndicatorItemValues["content"]=str(item.find('content').text) + else: + IndicatorItemValues["content"]="" + if item.find('comment'): + IndicatorItemValues["comment"]=str(item.find('comment').text) + else: + IndicatorItemValues["comment"]="" + + + jsonAttribute = create_attribute_json(IndicatorItemValues["context"],IndicatorItemValues["content"],IndicatorItemValues["comment"]) + attributes.append(jsonAttribute) + + return attributes + + +def create_misp_event_json(attributes): + import time + if iocDescriptions["authored_by"]: + attributes.append( + create_attribute_json(None,"authored_by",iocDescriptions["authored_by"],True) + ) + if iocDescriptions["authored_date"]: + attributes.append( + create_attribute_json(None,"authored_date",iocDescriptions["authored_date"],True) + ) + + ################################################## + # make short-description in "info field + # if not exist make description + # if "info"="short-description" make descrption as comment + mispInfoFild = "" + if iocDescriptions["short_description"]: + mispInfoFild = iocDescriptions["short_description"] + if iocDescriptions["description"]: + attributes.append( + create_attribute_json(None,"description",iocDescriptions["description"],True) + ) + else: + if iocDescriptions["description"]: + mispInfoFild = iocDescriptions["description"] + else: + mispInfoFild = "No description or short_description from IOC find." + + eventJson = { + "Event": { + "info": mispInfoFild, + "timestamp": "1", + "attribute_count": 0, + "analysis": "0", + "date": time.strftime("%Y-%m-%d"), + "org": "", + "distribution": "0", + "Attribute": [], + "proposal_email_lock": False, + "threat_level_id": "4", + } + } + + eventJson["Event"]["Attribute"] = attributes + + return eventJson + + +def get_descriptions(soup, description): + if soup.find(description.lower()): + return soup.find(description.lower()).text + return "" + +def save_ioc_description(soup): + list_description = ["short_description","authored_by","authored_date","description"] + + for description in list_description: + iocDescriptions[description]=get_descriptions(soup, description) + + return + + + +def get_taxonomy(soup): + import csv + taxonomy = [] + reader = csv.reader(open(csvTaxonomyFile, 'rb'), delimiter=';') + ##################################### + # save file in a dict + # r[0] = @link from csv + # r[1] = @value from csv + # = value + # r[2] = @keep + # 0 : don't creat tag + # 1 : tag created + # r[3] = @taxonomy + + csvdic = {i:r for i,r in enumerate(reader)} + + ######################################### + # find all link with soup + for n in soup.find_all('link', rel=True): + rel = str(n.attrs['rel'][0]).lower() + + ########################## + # build special taxo + # special string because link if a html value + relValue = str(n.next_sibling).strip() + if rel == 'family': + if len(relValue)>0: + taxonomy.append("malware_classification:malware-family='"+relValue+"'") + elif rel == 'threatgroup': + if len(relValue)>0: + taxonomy.append("malware_classification:malware-threatgroup='"+relValue+"'") + + ######################### + # build taxo from csv match + else: + taxo = [r[3] for r in + {i:r for i,r in csvdic.items() + if r[0].lower() == rel and str(r[2])=="1" + }.values() + if r[1].lower() == relValue.lower() and str(r[2])=="1" + ] + + # taxo find in correspondance file + if (len(taxo) > 0 and taxo[0] != '') : + taxonomy.append(taxo[0]) + # not find + return taxonomy + +def custum_color_tag(tagg): + color="#00ace6" + if ":amber" in tagg :color="#ffc200" + if ":green:" in tagg :color="#009933" + if "tlp:green" in tagg :color="#009933" + if ":red:" in tagg :color="#ff0000" + if "tlp:red" in tagg :color="#ff0000" + if "tlp:white" in tagg :color="#fafafa" + return color + +def push_event_to_misp(jsonEvent): + global misp + + #################### + # upload json event + r = misp.add_event(jsonEvent) + event=r.json() + + # save event id for file upload and tagg + iocDescriptions["misp_event_id"]=event["Event"]["id"] + + return + +def upload_file(): + + # filename,path, eid, distrib, ids, categ, info, ids, analysis, threat + misp.upload_sample( + iocDescriptions['filename'], + iocDescriptions["iocfile"], + iocDescriptions["misp_event_id"], + "0", + False, + "External analysis", + iocDescriptions["short_description"], + None, + "1", + "4", + ) + + return + +def update_tag(listOfTagg): + for tagg in listOfTagg: + color = custum_color_tag(tagg) + + ############################# + # creatz tag in MISP + + r = misp.new_tag(str(tagg), str(color)) + ############################# + # link tag to MISP event + toPost={} + toPost['Event']={'id':iocDescriptions["misp_event_id"]} + misp.add_tag( + toPost, + str(tagg)) + return + + +def main(): + global misp + global iocDescriptions + iocDescriptions = {} + + + ################################ + # parse for valid argments + parser = argparse.ArgumentParser(description='Get an event from a MISP instance.') + parser.add_argument("-i", "--input", required=True, help="Input file") + parser.add_argument("-t", "--tag", help="Add custom tags 'tlp:red,cossi:tmp=test'") + args = parser.parse_args() + + iocDescriptions["iocfile"]=os.path.abspath(args.input) + + ################################ + # check if file have ioc extention and if he is not empty + if check_valid_ioc(): + + ################################ + # Try to parse file + iocfileparse = get_parse_ioc_file() + else : + print "/!\ Bad format {0}".format(iocDescriptions["iocfile"]) + return + + ################################ + # save description for create event + save_ioc_description(iocfileparse) + + ################################ + # parse ioc and buid json attributes + jsonAttributes = create_attributes_from_ioc_json(iocfileparse) + + ################################ + # create a json misp event and append attributes + jsonEvent = create_misp_event_json(jsonAttributes) + + + ################################ + # try connection + try: + misp = misp_init(mispUrl, mispKey) + except: + print "/!\ Connection fail, bad url ({0}) or API key : {1}".format(mispUrl,mispKey) + return + + ################################ + # Add event to MSIP + push_event_to_misp(jsonEvent) + + + ################################ + # Upload the IOC file and close tmpfile + upload_file() + + ################################ + # Update MISP Event with tag from IOC + update_tag(get_taxonomy(iocfileparse)) + + ################################ + # Add custom Tag (-t) + if args.tag : + customTag = args.tag + update_tag(customTag.split(",")) + + + +if __name__ == '__main__': + main() diff --git a/examples/ioc-2-misp/keys.py.sample b/examples/ioc-2-misp/keys.py.sample new file mode 100644 index 0000000..5b73563 --- /dev/null +++ b/examples/ioc-2-misp/keys.py.sample @@ -0,0 +1,94 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +mispUrl = '' +mispKey = '' + +############################### +# file use for internal tag +# some sample can be find here : +# https://github.com/eset/malware-ioc +# https://github.com/fireeye/iocs +csvTaxonomyFile = "taxonomy.csv" + +# csv delimiter : ";" with quotechar : " + +############################### +# link sample + #~ + #~ APT + #~ APT12 + #~ Backdoor + #~ Apache 2.0 + #~ + +# @link from csv +# = rel attribut from +# @value from csv +# = value +# @keep +# 0 : don't create tag +# 1 : tag created +# @taxonomy +# define tag for misp +# @comment +# litte description but not use + + +######################################### +# https://www.circl.lu/doc/misp/categories-and-types/index.html +# /\ +# || +# || +# \/ +# http://schemas.mandiant.com/ + +# @index = Context/search form ioc +# @(1, 2, 3) +# 1. categorie mapping +# 2. type mapping +# 3. optionnal comment + + +iocMispMapping = { + + ('DriverItem/DriverName') : (u'Artifacts dropped',u'other', u'DriverName. '), + + ('DnsEntryItem/Host') : (u'Network activity',u'domain'), + + ('Email/To') : (u'Targeting data',u'target-email'), + ('Email/Date') : (u'Other',u'comment',u'EmailDate. '), + ('Email/Body') : (u'Payload delivery',u'email-subject'), + ('Email/From') : (u'Payload delivery',u'email-dst'), + ('Email/Subject') : (u'Payload delivery',u'email-subject'), + ('Email/Attachment/Name') : (u'Payload delivery',u'email-attachment'), + + ('FileItem/Md5sum') : (u'External analysis',u'md5'), + ('FileItem/Sha1sum') : (u'External analysis',u'sha1'), + ('FileItem/FileName') : (u'External analysis',u'filename'), + ('FileItem/FullPath') : (u'External analysis',u'filename'), + ('FileItem/FilePath') : (u'External analysis',u'filename'), + ('FileItem/Sha256sum') : (u'External analysis',u'sha256'), + + ('Network/URI') : (u'Network activity',u'uri'), + ('Network/DNS') : (u'Network activity',u'domain'), + ('Network/String') : (u'Network activity',u'ip-dst'), + ('Network/UserAgent') : (u'Network activity',u'user-agent'), + + ('PortItem/localIP') : (u'Network activity',u'ip-dst'), + + ('ProcessItem/name') : (u'External analysis',u'pattern-in-memory', u'ProcessName. '), + ('ProcessItem/path') : (u'External analysis',u'pattern-in-memory', u'ProcessPath. '), + ('ProcessItem/Mutex') : (u'Artifacts dropped',u'mutex', u'mutex'), + ('ProcessItem/Pipe/Name') : (u'Artifacts dropped',u'named pipe'), + ('ProcessItem/Mutex/Name') : (u'Artifacts dropped',u'mutex', u'MutexName. '), + + ('RegistryItem/Text') : (u'Artifacts dropped',u'regkey', u'RegistryText. '), + ('RegistryItem/Path') : (u'Artifacts dropped',u'regkey', u'RegistryPath. '), + + ('ServiceItem/name') : (u'Artifacts dropped',u'windows-service-name'), + ('ServiceItem/type') : (u'Artifacts dropped',u'pattern-in-memory', u'ServiceType. '), + + ('Snort/Snort') : (u'Network activity',u'snort'), + + } diff --git a/examples/ioc-2-misp/taxonomy.csv b/examples/ioc-2-misp/taxonomy.csv new file mode 100644 index 0000000..73ac977 --- /dev/null +++ b/examples/ioc-2-misp/taxonomy.csv @@ -0,0 +1,12 @@ +link,value,keep,taxonomy,comment +classification,TLP AMBER,1,tlp:amber, +classification,TLP GREEN,1,tlp:green, +confidential,TLP-AMBER,1,tlp:amber, +confidential,TLP GREEN,1,tlp:green, +confidential,TLP-GREEN,1,tlp:green, +confidential,TLP RED,1,tlp:red, +exportable,Yes,0,, +family,APT,1,malware_classification:malware-category='APT', +family,APT3,1,malware_classification:malware-category='APT3',https://github.com/fireeye/iocs/tree/master/APT3 +license,Apache 2.0,0,, +threatcategory,APT3,1,malware_classification:malware-category='APT3',https://github.com/fireeye/iocs/tree/master/APT3 From f93bad9564c719e6d99dd377d644121a50c5f30b Mon Sep 17 00:00:00 2001 From: ANSSI-BSO-D Date: Thu, 12 May 2016 17:35:05 +0200 Subject: [PATCH 041/223] form --- examples/ioc-2-misp/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/ioc-2-misp/README.md b/examples/ioc-2-misp/README.md index dc22d58..60412f6 100644 --- a/examples/ioc-2-misp/README.md +++ b/examples/ioc-2-misp/README.md @@ -4,9 +4,9 @@ Python script for ioc import to misp ### requires -> python 2.7 -> PyMISP -> BeautifulSoup (apt-get install python-bs4 lxml) +> python 2.7 +> PyMISP +> BeautifulSoup (apt-get install python-bs4 python-lxml) ### Usage @@ -22,4 +22,4 @@ time find /iocsample -type f|while read line ;do python ioc2misp.py -i ${line};d * rename keys.py.sample as keys.py * add your url and api key in keys.py - * use command in terminal \ No newline at end of file + * use command in terminal From 5108d78f8db88410a1376449a69455cc1bb90de9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Thu, 19 May 2016 14:09:01 +0200 Subject: [PATCH 042/223] add function get_sharing_groups --- pymisp/api.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pymisp/api.py b/pymisp/api.py index 9022d34..1440287 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1010,6 +1010,15 @@ class PyMISP(object): url = urljoin(self.root_url, 'sightings/add/') return session.post(url, data=json.dumps(jdata)) + # ############## Sharing Groups ################## + + def get_sharing_groups(self): + session = self.__prepare_session(force_out=None) + url = urljoin(self.root_url, 'sharing_groups/index.json') + response = session.get(url) + return self._check_response(response)['response'][0] + + # ############## Deprecated (Pure XML API should not be used) ################## @deprecated def download_all(self): From 70808a46ca1ec1381f32e96764063a90244b8f34 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Thu, 19 May 2016 14:30:43 +0200 Subject: [PATCH 043/223] File indention fixed --- pymisp/api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 63d64c1..07e65dc 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1023,9 +1023,9 @@ class PyMISP(object): def get_sharing_groups(self): session = self.__prepare_session(force_out=None) - url = urljoin(self.root_url, 'sharing_groups/index.json') - response = session.get(url) - return self._check_response(response)['response'][0] + url = urljoin(self.root_url, 'sharing_groups/index.json') + response = session.get(url) + return self._check_response(response)['response'][0] # ############## Deprecated (Pure XML API should not be used) ################## From d781b4690fe85710d2492fab3ebc8b2c2fc002dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Mon, 23 May 2016 15:16:31 +0200 Subject: [PATCH 044/223] add function get_tags_statistics --- pymisp/api.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 1440287..7860808 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -981,33 +981,49 @@ class PyMISP(object): def get_attributes_statistics(self, context='type', percentage=None, force_out=None): """ - Get statistics from the MISP instance + Get attributes statistics from the MISP instance """ session = self.__prepare_session(force_out) if (context != 'category'): - context = 'type' + context = 'type' if(percentage!=None): url = urljoin(self.root_url, 'attributes/attributeStatistics/{}/{}'.format(context, percentage)) else: url = urljoin(self.root_url, 'attributes/attributeStatistics/{}'.format(context)) return session.get(url).json() - # ############## Sightings ################## + def get_tags_statistics(self, percentage=None, name_sort=None, force_out=None): + """ + Get tags statistics from the MISP instance + """ + session = self.__prepare_session(force_out) + if (percentage != None): + percentage = 'true' + else: + percentage = 'false' + if (name_sort != None): + name_sort = 'true' + else: + name_sort = 'false' + url = urljoin(self.root_url, 'tags/tagStatistics/{}/{}'.format(percentage, name_sort)) + return session.get(url).json() + +# ############## Sightings ################## def sighting_per_id(self, attribute_id, force_out=None): session = self.__prepare_session(force_out) - url = urljoin(self.root_url, 'sightings/add/{}'.format(attribute_id)) + url = urljoin(self.root_url, 'sightings/add/{}'.format(attribute_id)) return session.post(url) def sighting_per_uuid(self, attribute_uuid, force_out=None): session = self.__prepare_session(force_out) - url = urljoin(self.root_url, 'sightings/add/{}'.format(attribute_uuid)) + url = urljoin(self.root_url, 'sightings/add/{}'.format(attribute_uuid)) return session.post(url) def sighting_per_json(self, json_file, force_out=None): session = self.__prepare_session(force_out) jdata = json.load(open(json_file)) - url = urljoin(self.root_url, 'sightings/add/') + url = urljoin(self.root_url, 'sightings/add/') return session.post(url, data=json.dumps(jdata)) # ############## Sharing Groups ################## From 03c2a053f41ea0035edd5af03e93eb9fb291346e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 9 Jun 2016 14:50:44 +0200 Subject: [PATCH 045/223] Tag version 2.4.48 --- pymisp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 18eac4f..af4e28c 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,3 +1,3 @@ -__version__ = '2.4.36' +__version__ = '2.4.48' from .api import PyMISP, PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey From 1da766093426c53bdeadb761e9b70d7b17dd0347 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 13 Jun 2016 19:14:32 +0900 Subject: [PATCH 046/223] Make pep8 happy --- examples/ioc-2-misp/ioc2misp.py | 542 ++++++++++++++++---------------- 1 file changed, 266 insertions(+), 276 deletions(-) mode change 100644 => 100755 examples/ioc-2-misp/ioc2misp.py diff --git a/examples/ioc-2-misp/ioc2misp.py b/examples/ioc-2-misp/ioc2misp.py old mode 100644 new mode 100755 index ecbeb60..5becc74 --- a/examples/ioc-2-misp/ioc2misp.py +++ b/examples/ioc-2-misp/ioc2misp.py @@ -8,341 +8,331 @@ from keys import mispUrl, mispKey, csvTaxonomyFile, iocMispMapping try: - from pymisp import PyMISP + from pymisp import PyMISP except: - print "you need pymisp form github" - import sys - sys.exit(1) - -import json + print("you need pymisp form github") + import sys + sys.exit(1) + import os import argparse try: - from bs4 import BeautifulSoup + from bs4 import BeautifulSoup except: - print "install BeautifulSoup : sudo apt-get install python-bs4 python-lxml" - import sys - sys.exit(1) + print("install BeautifulSoup : sudo apt-get install python-bs4 python-lxml") + import sys + sys.exit(1) + def misp_init(url, key): - return PyMISP(url, key, False, 'json') - -def check_valid_ioc(): - - (filepath, filename) = os.path.split(iocDescriptions["iocfile"]) - (shortname, extension) = os.path.splitext(filename) + return PyMISP(url, key, False, 'json') + + +def check_valid_ioc(): + + (filepath, filename) = os.path.split(iocDescriptions["iocfile"]) + (shortname, extension) = os.path.splitext(filename) + + if (("ioc" in extension)) and (sum(1 for _ in open(iocDescriptions["iocfile"])) > 1): + iocDescriptions['filename'] = filename + return True + return False - if (("ioc" in extension)) and (sum(1 for _ in open(iocDescriptions["iocfile"])) > 1): - iocDescriptions['filename'] = filename - return True - return False def get_parse_ioc_file(): + return BeautifulSoup(open(iocDescriptions["iocfile"]), "lxml") - return BeautifulSoup(open(iocDescriptions["iocfile"]) , "lxml") def parse_ioc_search_content(iocContextSearch): - for k,v in iocMispMapping.items(): - if str(k).lower() == str(iocContextSearch).lower(): - return v - return False - -def create_attribute_json(iocContextSearch, attributeValue, attributeComment,force=False): - ##################################### - # force used for description to upload - if force: - parseResult=("Other","comment") - else: - parseResult = parse_ioc_search_content(iocContextSearch) - - if parseResult is False: - - print "/!\ Not implemented :: {0} :: {1} :: Item add as 'Other','Comment'. Add it in your keys.py".format(iocContextSearch,attributeValue) - ######################################## - # force import to misp - parseResult=("Other","comment") - - comment = "" - try : - comment= parseResult[2]+attributeComment - except: - comment= attributeComment + for k, v in iocMispMapping.items(): + if str(k).lower() == str(iocContextSearch).lower(): + return v + return False + + +def create_attribute_json(iocContextSearch, attributeValue, attributeComment, force=False): + ##################################### + # force used for description to upload + if force: + parseResult = ("Other", "comment") + else: + parseResult = parse_ioc_search_content(iocContextSearch) + + if parseResult is False: + + print("/!\ Not implemented :: {0} :: {1} :: Item add as 'Other','Comment'. Add it in your keys.py".format(iocContextSearch, attributeValue)) + ######################################## + # force import to misp + parseResult = ("Other", "comment") + + comment = "" + try: + comment = parseResult[2] + attributeComment + except: + comment = attributeComment + + attribute = {"category": parseResult[0], + "type": parseResult[1], + "value": attributeValue, + "timestamp": "0", + "to_ids": "0", + "distribution": "0", + "comment": comment + } + return attribute - attribute = { - "category": parseResult[0], - "type": parseResult[1], - "value": attributeValue, - "timestamp": "0", - "to_ids": "0", - "distribution": "0", - "comment": comment, - } - return attribute def create_attributes_from_ioc_json(soup): - attributes = [] + attributes = [] - IndicatorItemValues = {} - for item in soup.find_all("indicatoritem"): + IndicatorItemValues = {} + for item in soup.find_all("indicatoritem"): - if item.find('context'): - IndicatorItemValues["context"]=str(item.find('context')['search']) - else: - IndicatorItemValues["context"]="" - if item.find('content'): - IndicatorItemValues["content"]=str(item.find('content').text) - else: - IndicatorItemValues["content"]="" - if item.find('comment'): - IndicatorItemValues["comment"]=str(item.find('comment').text) - else: - IndicatorItemValues["comment"]="" - + if item.find('context'): + IndicatorItemValues["context"] = str(item.find('context')['search']) + else: + IndicatorItemValues["context"] = "" + if item.find('content'): + IndicatorItemValues["content"] = str(item.find('content').text) + else: + IndicatorItemValues["content"] = "" + if item.find('comment'): + IndicatorItemValues["comment"] = str(item.find('comment').text) + else: + IndicatorItemValues["comment"] = "" - jsonAttribute = create_attribute_json(IndicatorItemValues["context"],IndicatorItemValues["content"],IndicatorItemValues["comment"]) - attributes.append(jsonAttribute) + jsonAttribute = create_attribute_json(IndicatorItemValues["context"], IndicatorItemValues["content"], IndicatorItemValues["comment"]) + attributes.append(jsonAttribute) - return attributes + return attributes def create_misp_event_json(attributes): - import time - if iocDescriptions["authored_by"]: - attributes.append( - create_attribute_json(None,"authored_by",iocDescriptions["authored_by"],True) - ) - if iocDescriptions["authored_date"]: - attributes.append( - create_attribute_json(None,"authored_date",iocDescriptions["authored_date"],True) - ) - - ################################################## - # make short-description in "info field - # if not exist make description - # if "info"="short-description" make descrption as comment - mispInfoFild = "" - if iocDescriptions["short_description"]: - mispInfoFild = iocDescriptions["short_description"] - if iocDescriptions["description"]: - attributes.append( - create_attribute_json(None,"description",iocDescriptions["description"],True) - ) - else: - if iocDescriptions["description"]: - mispInfoFild = iocDescriptions["description"] - else: - mispInfoFild = "No description or short_description from IOC find." - - eventJson = { - "Event": { - "info": mispInfoFild, - "timestamp": "1", - "attribute_count": 0, - "analysis": "0", - "date": time.strftime("%Y-%m-%d"), - "org": "", - "distribution": "0", - "Attribute": [], - "proposal_email_lock": False, - "threat_level_id": "4", - } - } + import time + if iocDescriptions["authored_by"]: + attributes.append(create_attribute_json(None, "authored_by", iocDescriptions["authored_by"], True)) + if iocDescriptions["authored_date"]: + attributes.append(create_attribute_json(None, "authored_date", iocDescriptions["authored_date"], True)) - eventJson["Event"]["Attribute"] = attributes + ################################################## + # make short-description in "info field + # if not exist make description + # if "info"="short-description" make descrption as comment + mispInfoFild = "" + if iocDescriptions["short_description"]: + mispInfoFild = iocDescriptions["short_description"] + if iocDescriptions["description"]: + attributes.append(create_attribute_json(None, "description", iocDescriptions["description"], True)) + else: + if iocDescriptions["description"]: + mispInfoFild = iocDescriptions["description"] + else: + mispInfoFild = "No description or short_description from IOC find." - return eventJson + eventJson = {"Event": {"info": mispInfoFild, + "timestamp": "1", + "attribute_count": 0, + "analysis": "0", + "date": time.strftime("%Y-%m-%d"), + "org": "", + "distribution": "0", + "Attribute": [], + "proposal_email_lock": False, + "threat_level_id": "4", + }} + + eventJson["Event"]["Attribute"] = attributes + + return eventJson def get_descriptions(soup, description): - if soup.find(description.lower()): - return soup.find(description.lower()).text - return "" - + if soup.find(description.lower()): + return soup.find(description.lower()).text + return "" + + def save_ioc_description(soup): - list_description = ["short_description","authored_by","authored_date","description"] - - for description in list_description: - iocDescriptions[description]=get_descriptions(soup, description) + list_description = ["short_description", "authored_by", "authored_date", "description"] - return + for description in list_description: + iocDescriptions[description] = get_descriptions(soup, description) + return def get_taxonomy(soup): - import csv - taxonomy = [] - reader = csv.reader(open(csvTaxonomyFile, 'rb'), delimiter=';') - ##################################### - # save file in a dict - # r[0] = @link from csv - # r[1] = @value from csv - # = value - # r[2] = @keep - # 0 : don't creat tag - # 1 : tag created - # r[3] = @taxonomy + import csv + taxonomy = [] + reader = csv.reader(open(csvTaxonomyFile, 'rb'), delimiter=';') + ##################################### + # save file in a dict + # r[0] = @link from csv + # r[1] = @value from csv + # = value + # r[2] = @keep + # 0 : don't creat tag + # 1 : tag created + # r[3] = @taxonomy - csvdic = {i:r for i,r in enumerate(reader)} + csvdic = {i: r for i, r in enumerate(reader)} + + ######################################### + # find all link with soup + for n in soup.find_all('link', rel=True): + rel = str(n.attrs['rel'][0]).lower() + + ########################## + # build special taxo + # special string because link if a html value + relValue = str(n.next_sibling).strip() + if rel == 'family': + if len(relValue) > 0: + taxonomy.append("malware_classification:malware-family='" + relValue + "'") + elif rel == 'threatgroup': + if len(relValue) > 0: + taxonomy.append("malware_classification:malware-threatgroup='" + relValue + "'") + + ######################### + # build taxo from csv match + else: + taxo = [r[3] for r in {i: r for i, r in csvdic.items() if r[0].lower() == rel and str(r[2]) == "1"}.values() if r[1].lower() == relValue.lower() and str(r[2]) == "1"] + + # taxo find in correspondance file + if (len(taxo) > 0 and taxo[0] != ''): + taxonomy.append(taxo[0]) + # not find + return taxonomy - ######################################### - # find all link with soup - for n in soup.find_all('link', rel=True): - rel = str(n.attrs['rel'][0]).lower() - - ########################## - # build special taxo - # special string because link if a html value - relValue = str(n.next_sibling).strip() - if rel == 'family': - if len(relValue)>0: - taxonomy.append("malware_classification:malware-family='"+relValue+"'") - elif rel == 'threatgroup': - if len(relValue)>0: - taxonomy.append("malware_classification:malware-threatgroup='"+relValue+"'") - - ######################### - # build taxo from csv match - else: - taxo = [r[3] for r in - {i:r for i,r in csvdic.items() - if r[0].lower() == rel and str(r[2])=="1" - }.values() - if r[1].lower() == relValue.lower() and str(r[2])=="1" - ] - - # taxo find in correspondance file - if (len(taxo) > 0 and taxo[0] != '') : - taxonomy.append(taxo[0]) - # not find - return taxonomy def custum_color_tag(tagg): - color="#00ace6" - if ":amber" in tagg :color="#ffc200" - if ":green:" in tagg :color="#009933" - if "tlp:green" in tagg :color="#009933" - if ":red:" in tagg :color="#ff0000" - if "tlp:red" in tagg :color="#ff0000" - if "tlp:white" in tagg :color="#fafafa" - return color + color = "#00ace6" + if ":amber" in tagg: + color = "#ffc200" + if ":green:" in tagg: + color = "#009933" + if "tlp:green" in tagg: + color = "#009933" + if ":red:" in tagg: + color = "#ff0000" + if "tlp:red" in tagg: + color = "#ff0000" + if "tlp:white" in tagg: + color = "#fafafa" + return color + def push_event_to_misp(jsonEvent): - global misp + global misp - #################### - # upload json event - r = misp.add_event(jsonEvent) - event=r.json() - - # save event id for file upload and tagg - iocDescriptions["misp_event_id"]=event["Event"]["id"] + #################### + # upload json event + r = misp.add_event(jsonEvent) + event = r.json() + + # save event id for file upload and tagg + iocDescriptions["misp_event_id"] = event["Event"]["id"] + + return - return def upload_file(): - # filename,path, eid, distrib, ids, categ, info, ids, analysis, threat - misp.upload_sample( - iocDescriptions['filename'], - iocDescriptions["iocfile"], - iocDescriptions["misp_event_id"], - "0", - False, - "External analysis", - iocDescriptions["short_description"], - None, - "1", - "4", - ) - - return + # filename,path, eid, distrib, ids, categ, info, ids, analysis, threat + misp.upload_sample(iocDescriptions['filename'], + iocDescriptions["iocfile"], + iocDescriptions["misp_event_id"], + "0", + False, + "External analysis", + iocDescriptions["short_description"], + None, + "1", + "4", + ) + return + def update_tag(listOfTagg): - for tagg in listOfTagg: - color = custum_color_tag(tagg) + for tagg in listOfTagg: + color = custum_color_tag(tagg) - ############################# - # creatz tag in MISP + ############################# + # creatz tag in MISP - r = misp.new_tag(str(tagg), str(color)) - ############################# - # link tag to MISP event - toPost={} - toPost['Event']={'id':iocDescriptions["misp_event_id"]} - misp.add_tag( - toPost, - str(tagg)) - return + misp.new_tag(str(tagg), str(color)) + ############################# + # link tag to MISP event + toPost = {} + toPost['Event'] = {'id': iocDescriptions["misp_event_id"]} + misp.add_tag(toPost, str(tagg)) + return def main(): - global misp - global iocDescriptions - iocDescriptions = {} - - - ################################ - # parse for valid argments - parser = argparse.ArgumentParser(description='Get an event from a MISP instance.') - parser.add_argument("-i", "--input", required=True, help="Input file") - parser.add_argument("-t", "--tag", help="Add custom tags 'tlp:red,cossi:tmp=test'") - args = parser.parse_args() + global misp + global iocDescriptions + iocDescriptions = {} - iocDescriptions["iocfile"]=os.path.abspath(args.input) - - ################################ - # check if file have ioc extention and if he is not empty - if check_valid_ioc(): - - ################################ - # Try to parse file - iocfileparse = get_parse_ioc_file() - else : - print "/!\ Bad format {0}".format(iocDescriptions["iocfile"]) - return + ################################ + # parse for valid argments + parser = argparse.ArgumentParser(description='Get an event from a MISP instance.') + parser.add_argument("-i", "--input", required=True, help="Input file") + parser.add_argument("-t", "--tag", help="Add custom tags 'tlp:red,cossi:tmp=test'") + args = parser.parse_args() - ################################ - # save description for create event - save_ioc_description(iocfileparse) - - ################################ - # parse ioc and buid json attributes - jsonAttributes = create_attributes_from_ioc_json(iocfileparse) + iocDescriptions["iocfile"] = os.path.abspath(args.input) - ################################ - # create a json misp event and append attributes - jsonEvent = create_misp_event_json(jsonAttributes) - - - ################################ - # try connection - try: - misp = misp_init(mispUrl, mispKey) - except: - print "/!\ Connection fail, bad url ({0}) or API key : {1}".format(mispUrl,mispKey) - return - - ################################ - # Add event to MSIP - push_event_to_misp(jsonEvent) + ################################ + # check if file have ioc extention and if he is not empty + if check_valid_ioc(): - - ################################ - # Upload the IOC file and close tmpfile - upload_file() + ################################ + # Try to parse file + iocfileparse = get_parse_ioc_file() + else: + print("/!\ Bad format {0}".format(iocDescriptions["iocfile"])) + return - ################################ - # Update MISP Event with tag from IOC - update_tag(get_taxonomy(iocfileparse)) - - ################################ - # Add custom Tag (-t) - if args.tag : - customTag = args.tag - update_tag(customTag.split(",")) + ################################ + # save description for create event + save_ioc_description(iocfileparse) + ################################ + # parse ioc and buid json attributes + jsonAttributes = create_attributes_from_ioc_json(iocfileparse) + + ################################ + # create a json misp event and append attributes + jsonEvent = create_misp_event_json(jsonAttributes) + + ################################ + # try connection + try: + misp = misp_init(mispUrl, mispKey) + except: + print("/!\ Connection fail, bad url ({0}) or API key : {1}".format(mispUrl, mispKey)) + return + + ################################ + # Add event to MSIP + push_event_to_misp(jsonEvent) + + ################################ + # Upload the IOC file and close tmpfile + upload_file() + + ################################ + # Update MISP Event with tag from IOC + update_tag(get_taxonomy(iocfileparse)) + + ################################ + # Add custom Tag (-t) + if args.tag: + customTag = args.tag + update_tag(customTag.split(",")) if __name__ == '__main__': - main() + main() From 234de2f4d1fb0419531350479be0be480fd111e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 13 Jun 2016 19:15:04 +0900 Subject: [PATCH 047/223] Add tag script --- examples/sighting.py | 0 examples/tags.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) mode change 100644 => 100755 examples/sighting.py create mode 100755 examples/tags.py diff --git a/examples/sighting.py b/examples/sighting.py old mode 100644 new mode 100755 diff --git a/examples/tags.py b/examples/tags.py new file mode 100755 index 0000000..81a2d8e --- /dev/null +++ b/examples/tags.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pymisp import PyMISP +from keys import misp_url, misp_key +import argparse +import json + + +# Usage for pipe masters: ./last.py -l 5h | jq . + + +def init(url, key): + return PyMISP(url, key, True, 'json', True) + + +def get_tags(m): + result = m.get_all_tags(True) + r = result + print(json.dumps(r) + '\n') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Get tags from MISP instance.') + + args = parser.parse_args() + + misp = init(misp_url, misp_key) + + get_tags(misp) From 0cc5d9c982eb52398746d1bb77c197467b0238be Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 13 Jun 2016 17:20:40 +0200 Subject: [PATCH 048/223] Comment removed --- examples/tags.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/examples/tags.py b/examples/tags.py index 81a2d8e..adf5a8d 100755 --- a/examples/tags.py +++ b/examples/tags.py @@ -7,9 +7,6 @@ import argparse import json -# Usage for pipe masters: ./last.py -l 5h | jq . - - def init(url, key): return PyMISP(url, key, True, 'json', True) From fe8415dbc77e57388913c9b5827f666d16b54756 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 15 Jun 2016 09:44:36 +0900 Subject: [PATCH 049/223] Fix check MISP latest version --- pymisp/__init__.py | 2 +- pymisp/api.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index af4e28c..2b6f91f 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,3 +1,3 @@ -__version__ = '2.4.48' +__version__ = '2.4.48.1' from .api import PyMISP, PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey diff --git a/pymisp/api.py b/pymisp/api.py index 4250054..c261b91 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -971,7 +971,7 @@ class PyMISP(object): """ Get the most recent version from github """ - r = requests.get('https://raw.githubusercontent.com/MISP/MISP/master/VERSION.json') + r = requests.get('https://raw.githubusercontent.com/MISP/MISP/2.4/VERSION.json') if r.status_code == 200: master_version = json.loads(r.text) return {'version': '{}.{}.{}'.format(master_version['major'], master_version['minor'], master_version['hotfix'])} From a42be22f781c3f43d2e6959cf55f76ea8f58a281 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 15 Jun 2016 11:44:08 +0900 Subject: [PATCH 050/223] Make pep8 happy --- pymisp/api.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index c261b91..c80d650 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -995,7 +995,7 @@ class PyMISP(object): session = self.__prepare_session(force_out) if (context != 'category'): context = 'type' - if(percentage!=None): + if percentage is not None: url = urljoin(self.root_url, 'attributes/attributeStatistics/{}/{}'.format(context, percentage)) else: url = urljoin(self.root_url, 'attributes/attributeStatistics/{}'.format(context)) @@ -1006,11 +1006,11 @@ class PyMISP(object): Get tags statistics from the MISP instance """ session = self.__prepare_session(force_out) - if (percentage != None): + if percentage is not None: percentage = 'true' else: percentage = 'false' - if (name_sort != None): + if name_sort is not None: name_sort = 'true' else: name_sort = 'false' @@ -1040,10 +1040,9 @@ class PyMISP(object): def get_sharing_groups(self): session = self.__prepare_session(force_out=None) url = urljoin(self.root_url, 'sharing_groups/index.json') - response = session.get(url) + response = session.get(url) return self._check_response(response)['response'][0] - # ############## Deprecated (Pure XML API should not be used) ################## @deprecated def download_all(self): @@ -1068,5 +1067,3 @@ class PyMISP(object): template = urljoin(self.root_url, 'events/xml/download/{}/{}'.format(event_id, attach)) session = self.__prepare_session('xml') return session.get(template) - - From 8241d4ce9345264c153ebb4caf77424d722b4022 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 16 Jun 2016 13:48:40 +0900 Subject: [PATCH 051/223] Fix python3 compat. Make Pep8 happy. --- .gitignore | 1 + examples/feed-generator/generate.py | 58 +++++++++++++---------------- 2 files changed, 27 insertions(+), 32 deletions(-) diff --git a/.gitignore b/.gitignore index 4b22c47..d8e1bbd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *.pyc examples/keys.py examples/cudeso.py +examples/feed-generator/output/*.json build/* dist/* pymisp.egg-info/* diff --git a/examples/feed-generator/generate.py b/examples/feed-generator/generate.py index 361fed3..e980ff0 100755 --- a/examples/feed-generator/generate.py +++ b/examples/feed-generator/generate.py @@ -1,29 +1,24 @@ -#!/usr/bin/python +#!/usr/bin/env python # -*- coding: utf-8 -*- import sys import json import os from pymisp import PyMISP -from settings import * +from settings import url, key, ssl, outputdir, filters, valid_attribute_distribution_levels -objectsToSave = { - 'Orgc': { - 'fields': ['name', 'uuid'], - 'multiple': False, - }, - 'Tag': { - 'fields': ['name', 'colour', 'exportable'], - 'multiple': True, - }, - 'Attribute': { - 'fields': ['uuid', 'value', 'category', 'type', - 'comment', 'data', 'timestamp', - 'to_ids'], - 'multiple': True, - }, - } +objectsToSave = {'Orgc': {'fields': ['name', 'uuid'], + 'multiple': False, + }, + 'Tag': {'fields': ['name', 'colour', 'exportable'], + 'multiple': True, + }, + 'Attribute': {'fields': ['uuid', 'value', 'category', 'type', + 'comment', 'data', 'timestamp', 'to_ids'], + 'multiple': True, + }, + } fieldsToSave = ['uuid', 'info', 'threat_level_id', 'analysis', 'timestamp', 'publish_timestamp', 'published', @@ -43,15 +38,15 @@ def init(): def saveEvent(misp, uuid): - try: - event = misp.get_event(uuid) - event = __cleanUpEvent(event) - event = json.dumps(event) - eventFile = open(os.path.join(outputdir, uuid + '.json'), 'w') - eventFile.write(event) - eventFile.close() - except: + event = misp.get_event(uuid) + if not event.json().get('Event'): + print('Error while fetching event: {}'.format(event.json()['message'])) sys.exit('Could not create file for event ' + uuid + '.') + event = __cleanUpEvent(event) + event = json.dumps(event) + eventFile = open(os.path.join(outputdir, uuid + '.json'), 'w') + eventFile.write(event) + eventFile.close() def __cleanUpEvent(event): @@ -103,7 +98,8 @@ def saveManifest(manifest): manifestFile = open(os.path.join(outputdir, 'manifest.json'), 'w') manifestFile.write(json.dumps(manifest)) manifestFile.close() - except: + except Exception as e: + print(e) sys.exit('Could not create the manifest file.') @@ -112,8 +108,7 @@ def __addEventToManifest(event): for eventTag in event['EventTag']: tags.append({'name': eventTag['Tag']['name'], 'colour': eventTag['Tag']['colour']}) - return { - 'Orgc': event['Orgc'], + return {'Orgc': event['Orgc'], 'Tag': tags, 'info': event['info'], 'date': event['date'], @@ -138,8 +133,7 @@ if __name__ == '__main__': for event in events: saveEvent(misp, event['uuid']) manifest[event['uuid']] = __addEventToManifest(event) - print "Event " + str(counter) + "/" + str(total) + " exported." + print("Event " + str(counter) + "/" + str(total) + " exported.") counter += 1 saveManifest(manifest) - print 'Manifest saved. Feed creation completed.' - + print('Manifest saved. Feed creation completed.') From 444565234600181bcaff5cfa7e9a783bb0310a76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Tue, 21 Jun 2016 15:46:09 +0200 Subject: [PATCH 052/223] Add example "create attributes distribution treemap" --- examples/treemap/README.md | 11 ++ examples/treemap/style.css | 46 ++++++ examples/treemap/test_attribute_treemap.html | 26 ++++ examples/treemap/tools.py | 153 +++++++++++++++++++ examples/treemap/treemap.py | 48 ++++++ 5 files changed, 284 insertions(+) create mode 100644 examples/treemap/README.md create mode 100644 examples/treemap/style.css create mode 100644 examples/treemap/test_attribute_treemap.html create mode 100644 examples/treemap/tools.py create mode 100644 examples/treemap/treemap.py diff --git a/examples/treemap/README.md b/examples/treemap/README.md new file mode 100644 index 0000000..bfcc99f --- /dev/null +++ b/examples/treemap/README.md @@ -0,0 +1,11 @@ +## Explanation + +* treemap.py is a script that will generate an interactive svg (attribute\_treemap.svg) containing a treepmap representing the distribution of attributes in a sample (data) fetched from the instance using "last" or "searchall" examples. +* It will also generate a html document with a table (attribute\_table.html) containing count for each type of attribute. +* test\_attribute\_treemap.html is a quick page made to visualize both treemap and table at the same time. + +## Requierements + +* [Pygal](https://github.com/Kozea/pygal/) + + diff --git a/examples/treemap/style.css b/examples/treemap/style.css new file mode 100644 index 0000000..8c5313b --- /dev/null +++ b/examples/treemap/style.css @@ -0,0 +1,46 @@ +body +{ + /*font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;*/ + font-family: Consolas, "Liberation Mono", Menlo, Courier, monospace; +} + +h1 +{ + font-size: 16px; + width: 290px; + text-align:center; +} + +/*** Stats Tables ***/ + +table +{ + border-collapse: collapse; + border-spacing: 0; + border: 1px solid #cbcbcb; +} + +tbody +{ + font-size:12px; +} + +table td +{ + border-left: 1px solid #cbcbcb; + border-width: 0 0 0 1px; + width: 150px; + margin: 0; + padding: 0.5em 1em; +} + + +table tr:nth-child(2n-1) td +{ + background-color: #f2f2f2; +} + +table tr td:first-child +{ + font-weight: bold; +} diff --git a/examples/treemap/test_attribute_treemap.html b/examples/treemap/test_attribute_treemap.html new file mode 100644 index 0000000..d6e8fc4 --- /dev/null +++ b/examples/treemap/test_attribute_treemap.html @@ -0,0 +1,26 @@ + + + + + + + + +
+ + + diff --git a/examples/treemap/tools.py b/examples/treemap/tools.py new file mode 100644 index 0000000..1b58bca --- /dev/null +++ b/examples/treemap/tools.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import json +from json import JSONDecoder +import random +import pygal +from pygal.style import Style +import pandas as pd + +################ Formatting ################ + +def eventsListBuildFromList(filename): + with open('testt', 'r') as myfile: + s=myfile.read().replace('\n', '') + decoder = JSONDecoder() + s_len = len(s) + Events = [] + end = 0 + while end != s_len: + Event, end = decoder.raw_decode(s, idx=end) + Events.append(Event) + data = [] + for e in Events: + data.append(pd.DataFrame.from_dict(e, orient='index')) + Events = pd.concat(data) + for it in range(Events['attribute_count'].size): + if Events['attribute_count'][it] == None: + Events['attribute_count'][it]='0' + else: + Events['attribute_count'][it]=int(Events['attribute_count'][it]) + Events = Events.set_index('id') + return Events + +def eventsListBuildFromArray(filename): + ''' + returns a structure listing all primary events in the sample + ''' + jdata = json.load(open(filename)) + jdata = jdata['response'] + Events = [] + for e in jdata: + Events.append(e) + data = [] + for e in Events: + data.append(pd.DataFrame.from_dict(e, orient='index')) + Events = pd.concat(data) + for it in range(Events['attribute_count'].size): + if Events['attribute_count'][it] == None: + Events['attribute_count'][it]='0' + else: + Events['attribute_count'][it]=int(Events['attribute_count'][it]) + Events = Events.set_index('id') + return Events + +def attributesListBuild(Events): + Attributes = [] + for Attribute in Events['Attribute']: + Attributes.append(pd.DataFrame(Attribute)) + return pd.concat(Attributes) + + +################ Basic Stats ################ + +def getNbAttributePerEventCategoryType(Attributes): + return Attributes.groupby(['event_id', 'category', 'type']).count()['id'] + + +################ Charts ################ + +def createStyle(indexlevels): + colorsList = [] + for i in range(len(indexlevels[0])): + colorsList.append("#%06X" % random.randint(0, 0xFFFFFF)) + style = Style( + background='transparent', + plot_background='#FFFFFF', + foreground='#111111', + foreground_strong='#111111', + foreground_subtle='#111111', + opacity='.6', + opacity_hover='.9', + transition='400ms ease-in', + colors=tuple(colorsList)) + return style, colorsList + +def createLabelsTreemap(indexlevels, indexlabels): + categories_levels = indexlevels[0] + cat = 0 + types = [] + cattypes = [] + categories_labels = indexlabels[0] + types_levels = indexlevels[1] + types_labels = indexlabels[1] + + for it in range(len(indexlabels[0])): + if categories_labels[it] != cat: + cattypes.append(types) + types = [] + cat += 1 + + types.append(types_levels[types_labels[it]]) + cattypes.append(types) + + return categories_levels, cattypes + + +def createTable(data, title, tablename, colorsList): + if tablename == None: + target = open('attribute_table.html', 'w') + else: + target = open(tablename, 'w') + target.truncate() + target.write('\n\n\n\n\n') + categories, types = createLabelsTreemap(data.index.levels, data.index.labels) + it = 0 + + for i in range(len(categories)): + table = pygal.Treemap(pretty_print=True) + target.write('\n

' + categories[i] + '

\n') + for typ in types[i]: + table.add(typ, data[it]) + it += 1 + target.write(table.render_table(transpose=True)) + target.write('\n\n') + target.close() + + +def createTreemap(data, title, treename = 'attribute_treemap.svg', tablename = 'attribute_table.html'): + style, colorsList = createStyle(data.index.levels) + treemap = pygal.Treemap(pretty_print=True, legend_at_bottom=True, style = style) + treemap.title = title + treemap.print_values = True + treemap.print_labels = True + + categories, types = createLabelsTreemap(data.index.levels, data.index.labels) + it = 0 + + for i in range(len(categories)): + types_labels = [] + for typ in types[i]: + tempdict = {} + tempdict['label'] = typ + tempdict['value'] = data[it] + types_labels.append(tempdict) + it += 1 + treemap.add(categories[i], types_labels) + + createTable(data, 'Attribute Distribution', tablename, colorsList) + if treename == None: + treemap.render_to_file('attribute_treemap.svg') + else: + treemap.render_to_file(treename) diff --git a/examples/treemap/treemap.py b/examples/treemap/treemap.py new file mode 100644 index 0000000..1e7ef63 --- /dev/null +++ b/examples/treemap/treemap.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pymisp import PyMISP +from keys import misp_url, misp_key, misp_verifycert +import argparse +import os +import json +import tools + +def init(url, key): + return PyMISP(url, key, misp_verifycert, 'json') + +########## fetch data ########## + +def searchall(m, search, url): + result = m.search_all(search) + with open('data', 'w') as f: + f.write(json.dumps(result)) + +def download_last(m, last): + result = m.download_last(last) + with open('data', 'w') as f: + f.write(json.dumps(result)) + + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py of searchall.py) and create a treemap epresenting the distribution of attributes in this sample.') + parser.add_argument("-f", "--function", required=True, help="The parameter can be either set to \"last\" or \"searchall\". If the parameter is not valid, \"last\" will be the default setting.") + parser.add_argument("-a", "--argument", required=True, help="if function is \"last\", time can be defined in days, hours, minutes (for example 5d or 12h or 30m). Otherwise, this argument is the string to search") + + args = parser.parse_args() + + misp = init(misp_url, misp_key) + + if args.function == "searchall": + searchall(misp, args.argument, misp_url) + else: + download_last(misp, args.argument) + + Events = tools.eventsListBuildFromArray('data') + + Attributes = tools.attributesListBuild(Events) + temp = tools.getNbAttributePerEventCategoryType(Attributes) + temp = temp.groupby(level=['category', 'type']).sum() + tools.createTreemap(temp, 'Attributes Distribution', 'attribute_treemap.svg', 'attribute_table.html') + From ec4b158c84599e16c60bf485c23a1957a690e7a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Tue, 21 Jun 2016 16:07:08 +0200 Subject: [PATCH 053/223] remove useless comments --- examples/treemap/style.css | 3 --- examples/treemap/test_attribute_treemap.html | 4 ---- 2 files changed, 7 deletions(-) diff --git a/examples/treemap/style.css b/examples/treemap/style.css index 8c5313b..03f1ec9 100644 --- a/examples/treemap/style.css +++ b/examples/treemap/style.css @@ -1,6 +1,5 @@ body { - /*font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;*/ font-family: Consolas, "Liberation Mono", Menlo, Courier, monospace; } @@ -11,8 +10,6 @@ h1 text-align:center; } -/*** Stats Tables ***/ - table { border-collapse: collapse; diff --git a/examples/treemap/test_attribute_treemap.html b/examples/treemap/test_attribute_treemap.html index d6e8fc4..8d6dd1b 100644 --- a/examples/treemap/test_attribute_treemap.html +++ b/examples/treemap/test_attribute_treemap.html @@ -18,9 +18,5 @@ - From b7c7afa4f63d23a4797e9936bdc302a650461c53 Mon Sep 17 00:00:00 2001 From: Antonio S Date: Mon, 27 Jun 2016 16:53:13 +0200 Subject: [PATCH 054/223] Added add_domain_ip attribute function --- pymisp/api.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pymisp/api.py b/pymisp/api.py index c80d650..1116a99 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -486,6 +486,11 @@ class PyMISP(object): attributes.append(self._prepare_full_attribute(category, 'domain', domain, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) + def add_domain_ip(self, event, domain, ip, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): + attributes = [] + attributes.append(self._prepare_full_attribute(category, 'domain|ip', "%s|%s" % (domain, ip), to_ids, comment, distribution)) + return self._send_attributes(event, attributes, proposal) + def add_url(self, event, url, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): attributes = [] attributes.append(self._prepare_full_attribute(category, 'url', url, to_ids, comment, distribution)) From 101a274447a2dd1e17b4c1b5a17aad1880dd5f56 Mon Sep 17 00:00:00 2001 From: Antonio S Date: Tue, 28 Jun 2016 13:12:37 +0200 Subject: [PATCH 055/223] Added function to AV detection link --- pymisp/api.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pymisp/api.py b/pymisp/api.py index 1116a99..60952d4 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -417,6 +417,11 @@ class PyMISP(object): return self._send_attributes(event, attributes, proposal) + def av_detection_link(self, event, link, category='Antivirus detection', to_ids=False, comment=None, distribution=None, proposal=False): + attributes = [] + attributes.append(self._prepare_full_attribute(category, 'link', link, to_ids, comment, distribution)) + return self._send_attributes(event, attributes, proposal) + def add_filename(self, event, filename, category='Artifacts dropped', to_ids=False, comment=None, distribution=None, proposal=False): attributes = [] attributes.append(self._prepare_full_attribute(category, 'filename', filename, to_ids, comment, distribution)) From 60b688ba0d014c253c38b98d56c2cd9e553ad065 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antonio=20S=C3=A1nchez?= Date: Tue, 28 Jun 2016 15:19:08 +0200 Subject: [PATCH 056/223] Update README.md with install instructions --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5a05cdb..c60cc2e 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,15 @@ PyMISP allows you to fetch events, add or update events/attributes, add or updat * [requests](http://docs.python-requests.org) -## Install +## Install from pip +~~~~ +pip install pymisp +~~~~ + +## Install the lastest version from repo ~~~~ +git clone https://github.com/CIRCL/PyMISP.git && cd PyMISP python setup.py install ~~~~ From 29476b6ebaa5b9fcb2c72e26d85a4eee958a46cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Fri, 1 Jul 2016 10:33:44 +0200 Subject: [PATCH 057/223] Add examples "create_dummy_event" and "create_massive_dummy_events" --- examples/events/README.md | 53 +++++++++++++++ examples/events/create_dummy_event.py | 23 +++++++ .../events/create_massive_dummy_events.py | 26 ++++++++ examples/events/dummy | 21 ++++++ examples/events/tools.py | 64 +++++++++++++++++++ pymisp/api.py | 2 +- 6 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 examples/events/README.md create mode 100644 examples/events/create_dummy_event.py create mode 100644 examples/events/create_massive_dummy_events.py create mode 100644 examples/events/dummy create mode 100644 examples/events/tools.py diff --git a/examples/events/README.md b/examples/events/README.md new file mode 100644 index 0000000..e53e6d6 --- /dev/null +++ b/examples/events/README.md @@ -0,0 +1,53 @@ +## Explanation + +This folder contains scripts made to create dummy events in order to test MISP instances. + +* dummy is a containing text only file used as uploaded attachement. +* create\_dummy\_event.py will create a given number of events (default: 1)with a randomly generated domain|ip attribute as well as a copy of dummy file. +* create\_massive\_dummy\_events.py will create a given number of events (default: 1) with a given number of randomly generated attributes(default: 3000). + +### Tools description + +* randomStringGenerator: generate a random string of a given size, characters used to build the string can be chosen, default are characters from string.ascii\_lowercase and string.digits +* randomIpGenerator: generate a random ip + +* floodtxt: add a generated string as attribute of the given event. The added attributes can be of the following category/type: + - Internal reference/comment + - Internal reference/text + - Internal reference/other + - Payload delivery/email-subject + - Artifact dropped/mutex + - Artifact dropped/filename +* floodip: add a generated ip as attribute of the given event. The added attributes can be of the following category/type: + - Network activity/ip-src + - Network activity/ip.dst +* flooddomain: add a generated domain-like string as attribute of the given event. The added attributes can be of the following category/type: + - Network activity/hostname + - Network activity/domain +* flooddomainip: add a generated domain|ip-like string as attribute of the given event. The added attribute is of the following category/type: + - Network activity/domain|ip +* floodemail: add a generated email-like string as attribute of the given event. The added attributes can be of the following category/type: + - Payload delivery/email-src + - Payload delivery/email-dst +* floodattachmentent: add a dummy file as attribute of the given event. The added attribute is of the following category/type: + - Payload delivery/attachment + +* create\_dummy\_event: create a dummy event named "dummy event" with these caracteristics: + - Distribution: Your organisation only + - Analysis: Initial + - Threat Level: Undefined + - Number of Attributes: 2 + - Attribute: + - category/type: Network activity/domain|ip + - value: Randomly generated + - Attribute: + -category/type: Payload delivery/attachment + - value: 'dummy' file +* create\_massive\_dummy\_events: create a dummy event named "massive dummy event" with these caracteristics: + - Distribution: Your organisation only + - Analysis: Initial + - Threat Level: Undefined + - Number of Attributes: Given as argument + - Attribute: + - category/type: Randomly chosen + - value: Randomly generated or dummy file diff --git a/examples/events/create_dummy_event.py b/examples/events/create_dummy_event.py new file mode 100644 index 0000000..63bd581 --- /dev/null +++ b/examples/events/create_dummy_event.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pymisp import PyMISP +from keys import misp_url, misp_key, misp_verifycert +import argparse +import tools + +def init(url, key): + return PyMISP(url, key, misp_verifycert, 'json') + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Create a given number of event containing an domain|ip attribute and an attachment each.') + parser.add_argument("-l", "--limit", type=int, help="Number of events to create (default 1)") + args = parser.parse_args() + + misp = init(misp_url, misp_key) + + if args.limit is None: + args.limit = 1 + + for i in range(args.limit): + tools.create_dummy_event(misp) diff --git a/examples/events/create_massive_dummy_events.py b/examples/events/create_massive_dummy_events.py new file mode 100644 index 0000000..192c782 --- /dev/null +++ b/examples/events/create_massive_dummy_events.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pymisp import PyMISP +from keys import misp_url, misp_key, misp_verifycert +import argparse +import tools + +def init(url, key): + return PyMISP(url, key, misp_verifycert, 'json') + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Create a given number of event containing a given number of attributes eachh.') + parser.add_argument("-l", "--limit", type=int, help="Number of events to create (default 1)") + parser.add_argument("-a", "--attribute", type=int, help="Number of attributes per event (default 3000)") + args = parser.parse_args() + + misp = init(misp_url, misp_key) + + if args.limit is None: + args.limit = 1 + if args.attribute is None: + args.attribute = 3000 + + for i in range(args.limit): + tools.create_massive_dummy_events(misp, args.attribute) diff --git a/examples/events/dummy b/examples/events/dummy new file mode 100644 index 0000000..9834857 --- /dev/null +++ b/examples/events/dummy @@ -0,0 +1,21 @@ +DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY +DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY +DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY +DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY +DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY +DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY +DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY +DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY +DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY +DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY +DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY +DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY +DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY +DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY +DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY +DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY +DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY +DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY +DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY +DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY +DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY diff --git a/examples/events/tools.py b/examples/events/tools.py new file mode 100644 index 0000000..893f777 --- /dev/null +++ b/examples/events/tools.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import random +from random import randint +import string + +def randomStringGenerator(size, chars=string.ascii_lowercase + string.digits): + return ''.join(random.choice(chars) for _ in range(size)) + +def randomIpGenerator(): + return str(randint(0, 255)) + '.' + str(randint(0, 255)) + '.' + str(randint(0, 255)) + '.' + str(randint(0, 255)) + +def floodtxt(misp, event, maxlength = 255): + text = randomStringGenerator(randint(1, maxlength)) + textfunctions = [misp.add_internal_comment, misp.add_internal_text, misp.add_internal_other, misp.add_email_subject, misp.add_mutex, misp.add_filename] + textfunctions[randint(0,5)](event, text) + +def floodip(misp, event): + ip = randomIpGenerator() + ipfunctions = [misp.add_ipsrc, misp.add_ipdst] + ipfunctions[randint(0,1)](event, ip) + +def flooddomain(misp, event, maxlength = 25): + a = randomStringGenerator(randint(1, maxlength)) + b = randomStringGenerator(randint(2, 3), chars=string.ascii_lowercase) + domain = a + '.' + b + domainfunctions = [misp.add_hostname, misp.add_domain] + domainfunctions[randint(0,1)](event, domain) + +def flooddomainip(misp, event, maxlength = 25): + a = randomStringGenerator(randint(1, maxlength)) + b = randomStringGenerator(randint(2, 3), chars=string.ascii_lowercase) + domain = a + '.' + b + ip = randomIpGenerator() + misp.add_domain_ip(event, domain, ip) + +def floodemail(misp, event, maxlength = 25): + a = randomStringGenerator(randint(1, maxlength)) + b = randomStringGenerator(randint(1, maxlength)) + c = randomStringGenerator(randint(2, 3), chars=string.ascii_lowercase) + email = a + '@' + b + '.'+ c + emailfunctions = [misp.add_email_src, misp.add_email_dst] + emailfunctions[randint(0,1)](event, email) + +def floodattachment(misp, eventid, it, distribution, to_ids, category, comment, info, analysis, threat_level_id): + filename = 'dummy' + str(it) + misp.upload_sample(filename, 'dummy', eventid, distribution, to_ids, category, comment, info, analysis, threat_level_id) + +def create_dummy_event(misp): + event = misp.new_event(0, 4, 0, 'dummy event') + flooddomainip(misp, event) + floodattachment(misp, event['Event']['id'], event['Event']['id'], event['Event']['distribution'], False, 'Payload delivery', '', event['Event']['info'], event['Event']['analysis'], event['Event']['threat_level_id']) + +def create_massive_dummy_events(misp, nbattribute): + event = misp.new_event(0, 4, 0, 'massive dummy event') + eventid = event['Event']['id'] + functions = [floodtxt, floodip, flooddomain, flooddomainip, floodemail, floodattachment] + for i in range(nbattribute): + choice = randint(0,5) + if choice == 5: + floodattachment(misp, eventid, i, event['Event']['distribution'], False, 'Payload delivery', '', event['Event']['info'], event['Event']['analysis'], event['Event']['threat_level_id']) + else: + functions[choice](misp,event) diff --git a/pymisp/api.py b/pymisp/api.py index 60952d4..fb1d38b 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1027,7 +1027,7 @@ class PyMISP(object): url = urljoin(self.root_url, 'tags/tagStatistics/{}/{}'.format(percentage, name_sort)) return session.get(url).json() -# ############## Sightings ################## + # ############## Sightings ################## def sighting_per_id(self, attribute_id, force_out=None): session = self.__prepare_session(force_out) From 0bf368b281541342ac557b6216c689e3ded698b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Fri, 1 Jul 2016 12:06:49 +0200 Subject: [PATCH 058/223] Random names for dummy files --- examples/events/tools.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/events/tools.py b/examples/events/tools.py index 893f777..322aace 100644 --- a/examples/events/tools.py +++ b/examples/events/tools.py @@ -43,8 +43,8 @@ def floodemail(misp, event, maxlength = 25): emailfunctions = [misp.add_email_src, misp.add_email_dst] emailfunctions[randint(0,1)](event, email) -def floodattachment(misp, eventid, it, distribution, to_ids, category, comment, info, analysis, threat_level_id): - filename = 'dummy' + str(it) +def floodattachment(misp, eventid, distribution, to_ids, category, comment, info, analysis, threat_level_id): + filename = randomStringGenerator(randint(1,128)) misp.upload_sample(filename, 'dummy', eventid, distribution, to_ids, category, comment, info, analysis, threat_level_id) def create_dummy_event(misp): @@ -59,6 +59,6 @@ def create_massive_dummy_events(misp, nbattribute): for i in range(nbattribute): choice = randint(0,5) if choice == 5: - floodattachment(misp, eventid, i, event['Event']['distribution'], False, 'Payload delivery', '', event['Event']['info'], event['Event']['analysis'], event['Event']['threat_level_id']) + floodattachment(misp, eventid, event['Event']['distribution'], False, 'Payload delivery', '', event['Event']['info'], event['Event']['analysis'], event['Event']['threat_level_id']) else: functions[choice](misp,event) From 13e0cd0901b705ccc8b744221b3f382b2db7a918 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 1 Jul 2016 13:52:31 +0200 Subject: [PATCH 059/223] Make scripts executable --- examples/events/create_dummy_event.py | 0 examples/events/create_massive_dummy_events.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 examples/events/create_dummy_event.py mode change 100644 => 100755 examples/events/create_massive_dummy_events.py diff --git a/examples/events/create_dummy_event.py b/examples/events/create_dummy_event.py old mode 100644 new mode 100755 diff --git a/examples/events/create_massive_dummy_events.py b/examples/events/create_massive_dummy_events.py old mode 100644 new mode 100755 From 836845abde53ee55bca93f098ece78880ab6b5c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 1 Jul 2016 14:21:48 +0200 Subject: [PATCH 060/223] Use same variable names as testing environment --- examples/events/create_massive_dummy_events.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/examples/events/create_massive_dummy_events.py b/examples/events/create_massive_dummy_events.py index 192c782..12a2826 100755 --- a/examples/events/create_massive_dummy_events.py +++ b/examples/events/create_massive_dummy_events.py @@ -2,12 +2,10 @@ # -*- coding: utf-8 -*- from pymisp import PyMISP -from keys import misp_url, misp_key, misp_verifycert +from keys import url, key import argparse import tools -def init(url, key): - return PyMISP(url, key, misp_verifycert, 'json') if __name__ == '__main__': parser = argparse.ArgumentParser(description='Create a given number of event containing a given number of attributes eachh.') @@ -15,7 +13,7 @@ if __name__ == '__main__': parser.add_argument("-a", "--attribute", type=int, help="Number of attributes per event (default 3000)") args = parser.parse_args() - misp = init(misp_url, misp_key) + misp = PyMISP(url, key, True, 'json') if args.limit is None: args.limit = 1 From 8c0b5b943fdb1004a36126fb89b0c35e99e3d7cc Mon Sep 17 00:00:00 2001 From: Deborah Servili Date: Tue, 5 Jul 2016 16:26:57 +0200 Subject: [PATCH 061/223] Rename examples/treemap/treemap.py to examples/statistics/attribute_treemap.py --- examples/{treemap/treemap.py => statistics/attribute_treemap.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/{treemap/treemap.py => statistics/attribute_treemap.py} (100%) diff --git a/examples/treemap/treemap.py b/examples/statistics/attribute_treemap.py similarity index 100% rename from examples/treemap/treemap.py rename to examples/statistics/attribute_treemap.py From 12849622ef9e88c174217cfa726af101ed20cd6d Mon Sep 17 00:00:00 2001 From: Deborah Servili Date: Wed, 6 Jul 2016 09:05:35 +0200 Subject: [PATCH 062/223] Rename examples/statistics/attribute_treemap.py to examples/treemap/treemap.py --- examples/{statistics/attribute_treemap.py => treemap/treemap.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/{statistics/attribute_treemap.py => treemap/treemap.py} (100%) diff --git a/examples/statistics/attribute_treemap.py b/examples/treemap/treemap.py similarity index 100% rename from examples/statistics/attribute_treemap.py rename to examples/treemap/treemap.py From 51a97255830318c45b2cc96a18076729d727150f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 11 Jul 2016 17:57:16 +0200 Subject: [PATCH 063/223] Add remove tag method --- pymisp/api.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pymisp/api.py b/pymisp/api.py index fb1d38b..35c8059 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -364,6 +364,13 @@ class PyMISP(object): return self._check_response(response) + def remove_tag(self, event, tag): + session = self.__prepare_session('json') + to_post = {'request': {'Event': {'id': event['Event']['id'], 'tag': tag}}} + response = session.post(urljoin(self.root_url, 'events/removeTag'), data=json.dumps(to_post)) + + return self._check_response(response) + def change_threat_level(self, event, threat_level_id): event['Event']['threat_level_id'] = threat_level_id self._prepare_update(event) From c97651e6ac93fcc23c9c263cd1a6200fffb04431 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 11 Jul 2016 18:54:17 +0200 Subject: [PATCH 064/223] Version bump --- pymisp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 2b6f91f..c6a29ba 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,3 +1,3 @@ -__version__ = '2.4.48.1' +__version__ = '2.4.48.2' from .api import PyMISP, PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey From caa8b963ec7727bfa05ce6375a8f69858e3a4598 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Wed, 13 Jul 2016 15:24:36 +0200 Subject: [PATCH 065/223] move files from examples/treemap to examples/situational-awareness/ --- examples/situational-awareness/README.md | 9 + .../attribute_treemap.py | 0 examples/situational-awareness/style.css | 46 ++++ examples/situational-awareness/tag_search.py | 69 +++++ examples/situational-awareness/tags_count.py | 70 +++++ .../test_attribute_treemap.html | 26 ++ examples/situational-awareness/tools.py | 245 ++++++++++++++++++ 7 files changed, 465 insertions(+) create mode 100644 examples/situational-awareness/README.md rename examples/{statistics => situational-awareness}/attribute_treemap.py (100%) create mode 100644 examples/situational-awareness/style.css create mode 100644 examples/situational-awareness/tag_search.py create mode 100644 examples/situational-awareness/tags_count.py create mode 100644 examples/situational-awareness/test_attribute_treemap.html create mode 100644 examples/situational-awareness/tools.py diff --git a/examples/situational-awareness/README.md b/examples/situational-awareness/README.md new file mode 100644 index 0000000..f0e4b19 --- /dev/null +++ b/examples/situational-awareness/README.md @@ -0,0 +1,9 @@ +## Explanation + +* treemap.py is a script that will generate an interactive svg (attribute\_treemap.svg) containing a treepmap representing the distribution of attributes in a sample (data) fetched from the instance using "last" or "searchall" examples. +* It will also generate a html document with a table (attribute\_table.html) containing count for each type of attribute. +* test\_attribute\_treemap.html is a quick page made to visualize both treemap and table at the same time. + +## Requierements + +* [Pygal](https://github.com/Kozea/pygal/) diff --git a/examples/statistics/attribute_treemap.py b/examples/situational-awareness/attribute_treemap.py similarity index 100% rename from examples/statistics/attribute_treemap.py rename to examples/situational-awareness/attribute_treemap.py diff --git a/examples/situational-awareness/style.css b/examples/situational-awareness/style.css new file mode 100644 index 0000000..8c5313b --- /dev/null +++ b/examples/situational-awareness/style.css @@ -0,0 +1,46 @@ +body +{ + /*font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;*/ + font-family: Consolas, "Liberation Mono", Menlo, Courier, monospace; +} + +h1 +{ + font-size: 16px; + width: 290px; + text-align:center; +} + +/*** Stats Tables ***/ + +table +{ + border-collapse: collapse; + border-spacing: 0; + border: 1px solid #cbcbcb; +} + +tbody +{ + font-size:12px; +} + +table td +{ + border-left: 1px solid #cbcbcb; + border-width: 0 0 0 1px; + width: 150px; + margin: 0; + padding: 0.5em 1em; +} + + +table tr:nth-child(2n-1) td +{ + background-color: #f2f2f2; +} + +table tr td:first-child +{ + font-weight: bold; +} diff --git a/examples/situational-awareness/tag_search.py b/examples/situational-awareness/tag_search.py new file mode 100644 index 0000000..a04f54a --- /dev/null +++ b/examples/situational-awareness/tag_search.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pymisp import PyMISP +from keys import misp_url, misp_key, misp_verifycert +from datetime import datetime +import argparse +import json +import tools + +def init(url, key): + return PyMISP(url, key, misp_verifycert, 'json') + +########## fetch data ########## + +def searchall(m, search, url): + result = m.search_all(search) + with open('data', 'w') as f: + f.write(json.dumps(result)) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py of searchall.py) and create a treemap epresenting the distribution of attributes in this sample.') + parser.add_argument("-s", "--search", help="string to search") + parser.add_argument("-t", "--tag", required=True, help="String to search in tags, can be composed. Example: \"ransomware|Ransomware\"") + parser.add_argument("-b", "--begindate", help="The research will look for Tags attached to events posted at or after the given startdate (format: yyyy-mm-dd): If no date is given, default time is epoch time (1970-1-1)") + parser.add_argument("-e", "--enddate", help="The research will look for Tags attached to events posted at or before the given enddate (format: yyyy-mm-dd): If no date is given, default time is now()") + + args = parser.parse_args() + + misp = init(misp_url, misp_key) + + searchall(misp, args.search, misp_url) + + if args.begindate is not None: + args.begindate = tools.toDatetime(args.begindate) + if args.enddate is not None: + args.enddate = tools.toDatetime(args.enddate) + + Events = tools.eventsListBuildFromArray('data') + TotalEvents = tools.getNbitems(Events) + Tags = tools.tagsListBuild(Events) + result = tools.isTagIn(Tags, args.tag) + TotalTags = len(result) + + Events = tools.selectInRange(Events, begin=args.begindate, end=args.enddate) + TotalPeriodEvents = tools.getNbitems(Events) + Tags = tools.tagsListBuild(Events) + result = tools.isTagIn(Tags, args.tag) + TotalPeriodTags = len(result) + + text = 'Studied pediod: from ' + if args.begindate is None: + text = text + '1970-01-01' + else: + text = text + str(args.begindate.date()) + text = text + ' to ' + if args.enddate is None: + text = text + str(datetime.now().date()) + else: + text = text + str(args.enddate.date()) + + print '\n========================================================' + print text + print 'During the studied pediod, ' + str(TotalPeriodTags) + ' events out of ' + str(TotalPeriodEvents) + ' contains at least one tag with ' + args.tag + '.' + if TotalTags != 0: + print 'It represents ' + str(round(100*TotalPeriodTags/TotalTags, 3)) + '% of the fetched events (' + str(TotalTags) + ') including this tag.' + if TotalEvents != 0: + print 'It also represents ' + str(round(100*TotalPeriodTags/TotalEvents, 3)) + '% of all the fetched events (' + str(TotalEvents) + ').' + diff --git a/examples/situational-awareness/tags_count.py b/examples/situational-awareness/tags_count.py new file mode 100644 index 0000000..cff5d9b --- /dev/null +++ b/examples/situational-awareness/tags_count.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pymisp import PyMISP +from keys import misp_url, misp_key, misp_verifycert +from datetime import datetime +import argparse +import json +import tools + +def init(url, key): + return PyMISP(url, key, misp_verifycert, 'json') + +########## fetch data ########## + +def searchall(m, search, url): + result = m.search_all(search) + with open('data', 'w') as f: + f.write(json.dumps(result)) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py of searchall.py) and create a treemap epresenting the distribution of attributes in this sample.') + parser.add_argument("-s", "--search", help="string to search") + parser.add_argument("-b", "--begindate", help="The research will look for Tags attached to events posted at or after the given startdate (format: yyyy-mm-dd): If no date is given, default time is epoch time (1970-1-1)") + parser.add_argument("-e", "--enddate", help="The research will look for Tags attached to events posted at or before the given enddate (format: yyyy-mm-dd): If no date is given, default time is now()") + + args = parser.parse_args() + + misp = init(misp_url, misp_key) + + if args.search is None: + args.search = '' + searchall(misp, args.search, misp_url) + + if args.begindate is not None: + args.begindate = tools.toDatetime(args.begindate) + if args.enddate is not None: + args.enddate = tools.toDatetime(args.enddate) + + Events = tools.eventsListBuildFromArray('data') + TotalEvents = tools.getNbitems(Events) + Tags = tools.tagsListBuild(Events) + result = tools.getNbOccurenceTags(Tags) + TotalTags = tools.getNbitems(Tags) + + Events = tools.selectInRange(Events, begin=args.begindate, end=args.enddate) + TotalPeriodEvents = tools.getNbitems(Events) + Tags = tools.tagsListBuild(Events) + result = tools.getNbOccurenceTags(Tags) + TotalPeriodTags = tools.getNbitems(Tags) + + text = 'Studied pediod: from ' + if args.begindate is None: + text = text + '1970-01-01' + else: + text = text + str(args.begindate.date()) + text = text + ' to ' + if args.enddate is None: + text = text + str(datetime.now().date()) + else: + text = text + str(args.enddate.date()) + + print '\n========================================================' + print text + print result + ''' + print 'During the studied pediod, ' + str(TotalPeriodTags) + ' events out of ' + str(TotalPeriodEvents) + ' contains at least one tag with ' + args.tag + '.' + print 'It represents ' + str(round(100*TotalPeriodTags/TotalTags,3)) + '% of the fetched events (' + str(TotalTags) + ') including this tag.' + print 'It also represents ' + str(round(100*TotalPeriodTags/TotalEvents,3)) + '% of all the fetched events (' + str(TotalEvents) + ').' + ''' diff --git a/examples/situational-awareness/test_attribute_treemap.html b/examples/situational-awareness/test_attribute_treemap.html new file mode 100644 index 0000000..d6e8fc4 --- /dev/null +++ b/examples/situational-awareness/test_attribute_treemap.html @@ -0,0 +1,26 @@ + + + + + + + + +
+ + + diff --git a/examples/situational-awareness/tools.py b/examples/situational-awareness/tools.py new file mode 100644 index 0000000..f259f9c --- /dev/null +++ b/examples/situational-awareness/tools.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import json +from json import JSONDecoder +import random +import pygal +from pygal.style import Style +import pandas as pd +from datetime import datetime +from datetime import timedelta +from dateutil.parser import parse + +################ Tools ################ + +def buildDoubleIndex(index1, index2, datatype): + it = -1 + newindex1 = [] + for index in index2: + if index == 0: + it+=1 + newindex1.append(index1[it]) + arrays = [newindex1, index2] + tuples = list(zip(*arrays)) + return pd.MultiIndex.from_tuples(tuples, names=['event', datatype]) + +def buildNewColumn(index2, column): + it = -1 + newcolumn = [] + for index in index2: + if index == 0: + it+=1 + newcolumn.append(column[it]) + return newcolumn + +def dateInRange(datetimeTested, begin=None, end=None): + if begin == None: + begin = datetime(1970,1,1) + if end == None: + end = datetime.now() + return begin <= datetimeTested <= end + +def addColumn(dataframe, columnList, columnName): + dataframe.loc[:, columnName] = pd.Series(columnList, index=dataframe.index) + +def dateInRange(datetimeTested, begin=None, end=None): + if begin == None: + begin = datetime(1970,1,1) + if end == None: + end = datetime.now() + return begin <= datetimeTested <= end + +def toDatetime(date): + temp = date.split('-') + return datetime(int(temp[0]), int(temp[1]), int(temp[2])) + +################ Formatting ################ + +def eventsListBuildFromList(filename): + with open('testt', 'r') as myfile: + s=myfile.read().replace('\n', '') + decoder = JSONDecoder() + s_len = len(s) + Events = [] + end = 0 + while end != s_len: + Event, end = decoder.raw_decode(s, idx=end) + Events.append(Event) + data = [] + for e in Events: + data.append(pd.DataFrame.from_dict(e, orient='index')) + Events = pd.concat(data) + for it in range(Events['attribute_count'].size): + if Events['attribute_count'][it] == None: + Events['attribute_count'][it]='0' + else: + Events['attribute_count'][it]=int(Events['attribute_count'][it]) + Events = Events.set_index('id') + return Events + +def eventsListBuildFromArray(filename): + ''' + returns a structure listing all primary events in the sample + ''' + jdata = json.load(open(filename)) + jdata = jdata['response'] + Events = [] + for e in jdata: + Events.append(e) + data = [] + for e in Events: + data.append(pd.DataFrame.from_dict(e, orient='index')) + Events = pd.concat(data) + for it in range(Events['attribute_count'].size): + if Events['attribute_count'][it] == None: + Events['attribute_count'][it]='0' + else: + Events['attribute_count'][it]=int(Events['attribute_count'][it]) + Events = Events.set_index('id') + return Events + +def attributesListBuild(Events): + Attributes = [] + for Attribute in Events['Attribute']: + Attributes.append(pd.DataFrame(Attribute)) + return pd.concat(Attributes) + +def tagsListBuild(Events): + Tags = [] + for Tag in Events['Tag']: + if type(Tag) is not list: + continue + Tags.append(pd.DataFrame(Tag)) + Tags = pd.concat(Tags) + columnDate = buildNewColumn(Tags.index, Events['date']) + addColumn(Tags, columnDate, 'date') + index = buildDoubleIndex(Events.index, Tags.index, 'tag') + Tags = Tags.set_index(index) + return Tags + +def selectInRange(Events, begin=None, end=None): + inRange = [] + for i, Event in Events.iterrows(): + if dateInRange(parse(Event['date']), begin, end): + inRange.append(Event.tolist()) + inRange = pd.DataFrame(inRange) + temp = Events.columns.tolist() + inRange.columns = temp + return inRange +''' +def isTagIn(dataframe, tag): + print 'tag =' + tag + result = [] + for tagname in dataframe['name']: + print tagname + if tag in tagname: + print 'True' + result.append(tagname) + return result +''' + +def isTagIn(dataframe, tag): + temp = Tags[Tags['name'].str.contains(test)].index.tolist() + index = [] + for i in range(len(temp)): + if temp[i][0] not in index: + index.append(temp[i][0]) + return index + +################ Basic Stats ################ + +def getNbitems(dataframe): + return len(dataframe.index) + +def getNbAttributePerEventCategoryType(Attributes): + return Attributes.groupby(['event_id', 'category', 'type']).count()['id'] + +def getNbOccurenceTags(Tags): + return Tags.groupby('name').count()['id'] + +################ Charts ################ + +def createStyle(indexlevels): + colorsList = [] + for i in range(len(indexlevels[0])): + colorsList.append("#%06X" % random.randint(0, 0xFFFFFF)) + style = Style( + background='transparent', + plot_background='#FFFFFF', + foreground='#111111', + foreground_strong='#111111', + foreground_subtle='#111111', + opacity='.6', + opacity_hover='.9', + transition='400ms ease-in', + colors=tuple(colorsList)) + return style, colorsList + +def createLabelsTreemap(indexlevels, indexlabels): + categories_levels = indexlevels[0] + cat = 0 + types = [] + cattypes = [] + categories_labels = indexlabels[0] + types_levels = indexlevels[1] + types_labels = indexlabels[1] + + for it in range(len(indexlabels[0])): + if categories_labels[it] != cat: + cattypes.append(types) + types = [] + cat += 1 + + types.append(types_levels[types_labels[it]]) + cattypes.append(types) + + return categories_levels, cattypes + + +def createTable(data, title, tablename, colorsList): + if tablename == None: + target = open('attribute_table.html', 'w') + else: + target = open(tablename, 'w') + target.truncate() + target.write('\n\n\n\n\n') + categories, types = createLabelsTreemap(data.index.levels, data.index.labels) + it = 0 + + for i in range(len(categories)): + table = pygal.Treemap(pretty_print=True) + target.write('\n

' + categories[i] + '

\n') + for typ in types[i]: + table.add(typ, data[it]) + it += 1 + target.write(table.render_table(transpose=True)) + target.write('\n\n') + target.close() + + +def createTreemap(data, title, treename = 'attribute_treemap.svg', tablename = 'attribute_table.html'): + style, colorsList = createStyle(data.index.levels) + treemap = pygal.Treemap(pretty_print=True, legend_at_bottom=True, style = style) + treemap.title = title + treemap.print_values = True + treemap.print_labels = True + + categories, types = createLabelsTreemap(data.index.levels, data.index.labels) + it = 0 + + for i in range(len(categories)): + types_labels = [] + for typ in types[i]: + tempdict = {} + tempdict['label'] = typ + tempdict['value'] = data[it] + types_labels.append(tempdict) + it += 1 + treemap.add(categories[i], types_labels) + + createTable(data, 'Attribute Distribution', tablename, colorsList) + if treename == None: + treemap.render_to_file('attribute_treemap.svg') + else: + treemap.render_to_file(treename) From 63fbf6ef5413f7dfec48641974f7d337a21ce81a Mon Sep 17 00:00:00 2001 From: Deborah Servili Date: Wed, 13 Jul 2016 16:36:54 +0200 Subject: [PATCH 066/223] Update tools.py Correct function isTagIn(dataframe, tag) --- examples/situational-awareness/tools.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/examples/situational-awareness/tools.py b/examples/situational-awareness/tools.py index f259f9c..8676dd8 100644 --- a/examples/situational-awareness/tools.py +++ b/examples/situational-awareness/tools.py @@ -127,20 +127,9 @@ def selectInRange(Events, begin=None, end=None): temp = Events.columns.tolist() inRange.columns = temp return inRange -''' -def isTagIn(dataframe, tag): - print 'tag =' + tag - result = [] - for tagname in dataframe['name']: - print tagname - if tag in tagname: - print 'True' - result.append(tagname) - return result -''' def isTagIn(dataframe, tag): - temp = Tags[Tags['name'].str.contains(test)].index.tolist() + temp = dataframe[dataframe['name'].str.contains(tag)].index.tolist() index = [] for i in range(len(temp)): if temp[i][0] not in index: From 414ddaec014dcbee5dbb9ec46c3bad890c1a06bc Mon Sep 17 00:00:00 2001 From: Hannah Ward Date: Thu, 14 Jul 2016 12:55:37 +0100 Subject: [PATCH 067/223] Added STIX retrieval - misp.get_stix(event_id=ID, with_attachment=True/False, from_date=YYYY-MM-DD, to_date=YYYY-MM-DD, tags=["tag1", "tag2"] ) --- .gitignore | 1 + pymisp/api.py | 22 ++++++++++++++++++++++ tests/test.py | 4 ++++ 3 files changed, 27 insertions(+) diff --git a/.gitignore b/.gitignore index d8e1bbd..4fb001d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +*.pem *.pyc examples/keys.py examples/cudeso.py diff --git a/pymisp/api.py b/pymisp/api.py index 35c8059..21d999c 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -217,6 +217,24 @@ class PyMISP(object): url = urljoin(self.root_url, 'events/{}'.format(event_id)) return session.get(url) + def get_stix_event(self, event_id=None, out_format="json", with_attachments=False, from_date=False, to_date=False, tags=False): + """ + Get an event/events in STIX format + """ + out_format = out_format.lower() + if tags: + if isinstance(tags, list): + tags = "&&".join(tags) + + session = self.__prepare_session(out_format) + url = urljoin(self.root_url, + "/events/stix/download/{}/{}/{}/{}/{}".format( + event_id, with_attachments, tags, from_date, to_date + )) + if self.debug: + print("Getting STIX event from {}".format(url)) + return session.get(url) + def add_event(self, event, force_out=None): """ Add a new event @@ -339,6 +357,10 @@ class PyMISP(object): response = self.get_event(int(eid), 'json') return self._check_response(response) + def get_stix(self, **kwargs): + response = self.get_stix_event(**kwargs) + return self._check_response(response) + def update(self, event): eid = event['Event']['id'] response = self.update_event(eid, event, 'json') diff --git a/tests/test.py b/tests/test.py index 4bd0b2f..5d7c5f1 100755 --- a/tests/test.py +++ b/tests/test.py @@ -109,6 +109,10 @@ class TestBasic(unittest.TestCase): event = self.misp.get_event(eventid) print event.json() + def get_stix(self, **kwargs): + event = self.misp.get_stix(kwargs) + print(event) + def add(self): event = {u'Event': {u'info': u'This is a test', u'locked': False, u'attribute_count': u'3', u'analysis': u'0', From b0a66da4deca32d0fb677d8ad38623b63902a319 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Thu, 21 Jul 2016 10:06:47 +0200 Subject: [PATCH 068/223] handling some NaN exceptions --- examples/situational-awareness/tools.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/examples/situational-awareness/tools.py b/examples/situational-awareness/tools.py index f259f9c..57ee710 100644 --- a/examples/situational-awareness/tools.py +++ b/examples/situational-awareness/tools.py @@ -3,6 +3,7 @@ import json from json import JSONDecoder +import math import random import pygal from pygal.style import Style @@ -57,7 +58,7 @@ def toDatetime(date): ################ Formatting ################ def eventsListBuildFromList(filename): - with open('testt', 'r') as myfile: + with open(filename, 'r') as myfile: s=myfile.read().replace('\n', '') decoder = JSONDecoder() s_len = len(s) @@ -92,7 +93,7 @@ def eventsListBuildFromArray(filename): data.append(pd.DataFrame.from_dict(e, orient='index')) Events = pd.concat(data) for it in range(Events['attribute_count'].size): - if Events['attribute_count'][it] == None: + if Events['attribute_count'][it] == None or (isinstance(Events['attribute_count'][it], float) and math.isnan(Events['attribute_count'][it])): Events['attribute_count'][it]='0' else: Events['attribute_count'][it]=int(Events['attribute_count'][it]) @@ -127,20 +128,9 @@ def selectInRange(Events, begin=None, end=None): temp = Events.columns.tolist() inRange.columns = temp return inRange -''' -def isTagIn(dataframe, tag): - print 'tag =' + tag - result = [] - for tagname in dataframe['name']: - print tagname - if tag in tagname: - print 'True' - result.append(tagname) - return result -''' def isTagIn(dataframe, tag): - temp = Tags[Tags['name'].str.contains(test)].index.tolist() + temp = dataframe[dataframe['name'].str.contains(tag)].index.tolist() index = [] for i in range(len(temp)): if temp[i][0] not in index: From 0f68ffc61755b3b5c1ee558b535d2a3c4d78ca86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Thu, 21 Jul 2016 10:09:10 +0200 Subject: [PATCH 069/223] modify fetching method to use last --- examples/situational-awareness/tag_search.py | 18 +++++++------- examples/situational-awareness/tags_count.py | 25 +++++++++----------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/examples/situational-awareness/tag_search.py b/examples/situational-awareness/tag_search.py index a04f54a..e862ce1 100644 --- a/examples/situational-awareness/tag_search.py +++ b/examples/situational-awareness/tag_search.py @@ -13,23 +13,25 @@ def init(url, key): ########## fetch data ########## -def searchall(m, search, url): - result = m.search_all(search) +def download_last(m, last): + result = m.download_last(last) with open('data', 'w') as f: f.write(json.dumps(result)) if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py of searchall.py) and create a treemap epresenting the distribution of attributes in this sample.') - parser.add_argument("-s", "--search", help="string to search") - parser.add_argument("-t", "--tag", required=True, help="String to search in tags, can be composed. Example: \"ransomware|Ransomware\"") - parser.add_argument("-b", "--begindate", help="The research will look for Tags attached to events posted at or after the given startdate (format: yyyy-mm-dd): If no date is given, default time is epoch time (1970-1-1)") - parser.add_argument("-e", "--enddate", help="The research will look for Tags attached to events posted at or before the given enddate (format: yyyy-mm-dd): If no date is given, default time is now()") + parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py) and give the number of occurrence of the given tag in this sample.') + parser.add_argument("-t", "--tag", required=True, help="tag to search (search for multiple tags is possible by using |. example : \"osint|OSINT\")") + parser.add_argument("-d", "--days", help="number of days before today to search. If not define, default value is 7") + parser.add_argument("-b", "--begindate", help="The research will look for tags attached to events posted at or after the given startdate (format: yyyy-mm-dd): If no date is given, default time is epoch time (1970-1-1)") + parser.add_argument("-e", "--enddate", help="The research will look for tags attached to events posted at or before the given enddate (format: yyyy-mm-dd): If no date is given, default time is now()") args = parser.parse_args() misp = init(misp_url, misp_key) - searchall(misp, args.search, misp_url) + if args.days is None: + args.days = '7' + download_last(misp, args.days + 'd') if args.begindate is not None: args.begindate = tools.toDatetime(args.begindate) diff --git a/examples/situational-awareness/tags_count.py b/examples/situational-awareness/tags_count.py index cff5d9b..80dfdcd 100644 --- a/examples/situational-awareness/tags_count.py +++ b/examples/situational-awareness/tags_count.py @@ -13,24 +13,26 @@ def init(url, key): ########## fetch data ########## -def searchall(m, search, url): - result = m.search_all(search) +def download_last(m, last): + result = m.download_last(last) with open('data', 'w') as f: f.write(json.dumps(result)) if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py of searchall.py) and create a treemap epresenting the distribution of attributes in this sample.') - parser.add_argument("-s", "--search", help="string to search") - parser.add_argument("-b", "--begindate", help="The research will look for Tags attached to events posted at or after the given startdate (format: yyyy-mm-dd): If no date is given, default time is epoch time (1970-1-1)") - parser.add_argument("-e", "--enddate", help="The research will look for Tags attached to events posted at or before the given enddate (format: yyyy-mm-dd): If no date is given, default time is now()") + parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py) and give the repartition of tags in this sample.') + parser.add_argument("-d", "--days", help="number of days before today to search. If not define, default value is 7") + parser.add_argument("-b", "--begindate", help="The research will look for tags attached to events posted at or after the given startdate (format: yyyy-mm-dd): If no date is given, default time is epoch time (1970-1-1)") + parser.add_argument("-e", "--enddate", help="The research will look for tags attached to events posted at or before the given enddate (format: yyyy-mm-dd): If no date is given, default time is now()") + + args = parser.parse_args() misp = init(misp_url, misp_key) - if args.search is None: - args.search = '' - searchall(misp, args.search, misp_url) + if args.days is None: + args.days = '7' + download_last(misp, args.days + 'd') if args.begindate is not None: args.begindate = tools.toDatetime(args.begindate) @@ -63,8 +65,3 @@ if __name__ == '__main__': print '\n========================================================' print text print result - ''' - print 'During the studied pediod, ' + str(TotalPeriodTags) + ' events out of ' + str(TotalPeriodEvents) + ' contains at least one tag with ' + args.tag + '.' - print 'It represents ' + str(round(100*TotalPeriodTags/TotalTags,3)) + '% of the fetched events (' + str(TotalTags) + ') including this tag.' - print 'It also represents ' + str(round(100*TotalPeriodTags/TotalEvents,3)) + '% of all the fetched events (' + str(TotalEvents) + ').' - ''' From ca5a4d0960d6dc49a9e9211b3a9bfc0c2cdfbfe8 Mon Sep 17 00:00:00 2001 From: Nils <3c7@users.noreply.github.com> Date: Thu, 21 Jul 2016 13:43:04 +0200 Subject: [PATCH 070/223] Fixes: expected bytes, got in download_samples() --- pymisp/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 21d999c..3ae0d12 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -917,10 +917,10 @@ class PyMISP(object): archive = zipfile.ZipFile(zipped) try: # New format - unzipped = BytesIO(archive.open(f['md5'], pwd='infected').read()) + unzipped = BytesIO(archive.open(f['md5'], pwd=b'infected').read()) except KeyError: # Old format - unzipped = BytesIO(archive.open(f['filename'], pwd='infected').read()) + unzipped = BytesIO(archive.open(f['filename'], pwd=b'infected').read()) details.append([f['event_id'], f['filename'], unzipped]) except zipfile.BadZipfile: # In case the sample isn't zipped From cd046d2f7afbd6fcd117553b5e18b6321ae06d36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Tue, 26 Jul 2016 11:05:20 +0200 Subject: [PATCH 071/223] Make printed date more consistent + update README.md --- examples/situational-awareness/README.md | 9 ++++ examples/situational-awareness/tag_search.py | 34 +++++++-------- examples/situational-awareness/tags_count.py | 23 ++++++---- examples/situational-awareness/tools.py | 46 +++++++++++++++++++- 4 files changed, 83 insertions(+), 29 deletions(-) diff --git a/examples/situational-awareness/README.md b/examples/situational-awareness/README.md index f0e4b19..58a1381 100644 --- a/examples/situational-awareness/README.md +++ b/examples/situational-awareness/README.md @@ -4,6 +4,15 @@ * It will also generate a html document with a table (attribute\_table.html) containing count for each type of attribute. * test\_attribute\_treemap.html is a quick page made to visualize both treemap and table at the same time. +* tags\_count.py is a script that count the number of occurences of every tags in a fetched sample of Events in a given period of time. +* tag\_search.py is a script that count the number of occurences of a given tag in a fetched sample of Events in a given period of time. + * Events will be fetched from _days_ days ago to today. + * _begindate_ is the beginning of the studied period. If it is later than today, an error will be raised. + * _enddate_ is the end of the studied period. If it is earlier than _begindate_, an error will be raised. + * tag\_search.py allows research for multiple tags is possible by separating each tag by the | symbol. + * Partial research is also possible with tag\_search.py. For instance, search for "ransom" will also return tags containin "ransomware". +:warning: These scripts are not time optimised + ## Requierements * [Pygal](https://github.com/Kozea/pygal/) diff --git a/examples/situational-awareness/tag_search.py b/examples/situational-awareness/tag_search.py index e862ce1..d695a00 100644 --- a/examples/situational-awareness/tag_search.py +++ b/examples/situational-awareness/tag_search.py @@ -21,7 +21,7 @@ def download_last(m, last): if __name__ == '__main__': parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py) and give the number of occurrence of the given tag in this sample.') parser.add_argument("-t", "--tag", required=True, help="tag to search (search for multiple tags is possible by using |. example : \"osint|OSINT\")") - parser.add_argument("-d", "--days", help="number of days before today to search. If not define, default value is 7") + parser.add_argument("-d", "--days", type=int, help="number of days before today to search. If not define, default value is 7") parser.add_argument("-b", "--begindate", help="The research will look for tags attached to events posted at or after the given startdate (format: yyyy-mm-dd): If no date is given, default time is epoch time (1970-1-1)") parser.add_argument("-e", "--enddate", help="The research will look for tags attached to events posted at or before the given enddate (format: yyyy-mm-dd): If no date is given, default time is now()") @@ -30,21 +30,22 @@ if __name__ == '__main__': misp = init(misp_url, misp_key) if args.days is None: - args.days = '7' - download_last(misp, args.days + 'd') + args.days = 7 + download_last(misp, str(args.days) + 'd') - if args.begindate is not None: - args.begindate = tools.toDatetime(args.begindate) - if args.enddate is not None: - args.enddate = tools.toDatetime(args.enddate) + tools.checkDateConsistancy(args.begindate, args.enddate, tools.getLastdate(args.days)) - Events = tools.eventsListBuildFromArray('data') - TotalEvents = tools.getNbitems(Events) - Tags = tools.tagsListBuild(Events) - result = tools.isTagIn(Tags, args.tag) - TotalTags = len(result) + if args.begindate is None: + args.begindate = tools.getLastdate(args.days) + else: + args.begindate = tools.setBegindate(tools.toDatetime(args.begindate), tools.getLastdate(args.days)) - Events = tools.selectInRange(Events, begin=args.begindate, end=args.enddate) + if args.enddate is None: + args.enddate = datetime.now() + else: + args.enddate = tools.setEnddate(tools.toDatetime(args.enddate)) + + Events = tools.selectInRange(tools.eventsListBuildFromArray('data'), begin=args.begindate, end=args.enddate) TotalPeriodEvents = tools.getNbitems(Events) Tags = tools.tagsListBuild(Events) result = tools.isTagIn(Tags, args.tag) @@ -64,8 +65,5 @@ if __name__ == '__main__': print '\n========================================================' print text print 'During the studied pediod, ' + str(TotalPeriodTags) + ' events out of ' + str(TotalPeriodEvents) + ' contains at least one tag with ' + args.tag + '.' - if TotalTags != 0: - print 'It represents ' + str(round(100*TotalPeriodTags/TotalTags, 3)) + '% of the fetched events (' + str(TotalTags) + ') including this tag.' - if TotalEvents != 0: - print 'It also represents ' + str(round(100*TotalPeriodTags/TotalEvents, 3)) + '% of all the fetched events (' + str(TotalEvents) + ').' - + if TotalPeriodEvents != 0: + print 'It represents ' + str(round(100*TotalPeriodTags/TotalPeriodEvents, 3)) + '% of the events in this period.' diff --git a/examples/situational-awareness/tags_count.py b/examples/situational-awareness/tags_count.py index 80dfdcd..58e6194 100644 --- a/examples/situational-awareness/tags_count.py +++ b/examples/situational-awareness/tags_count.py @@ -20,24 +20,29 @@ def download_last(m, last): if __name__ == '__main__': parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py) and give the repartition of tags in this sample.') - parser.add_argument("-d", "--days", help="number of days before today to search. If not define, default value is 7") + parser.add_argument("-d", "--days", type=int, help="number of days before today to search. If not define, default value is 7") parser.add_argument("-b", "--begindate", help="The research will look for tags attached to events posted at or after the given startdate (format: yyyy-mm-dd): If no date is given, default time is epoch time (1970-1-1)") parser.add_argument("-e", "--enddate", help="The research will look for tags attached to events posted at or before the given enddate (format: yyyy-mm-dd): If no date is given, default time is now()") - - args = parser.parse_args() misp = init(misp_url, misp_key) if args.days is None: - args.days = '7' - download_last(misp, args.days + 'd') + args.days = 7 + download_last(misp, str(args.days) + 'd') - if args.begindate is not None: - args.begindate = tools.toDatetime(args.begindate) - if args.enddate is not None: - args.enddate = tools.toDatetime(args.enddate) + tools.checkDateConsistancy(args.begindate, args.enddate, tools.getLastdate(args.days)) + + if args.begindate is None: + args.begindate = tools.getLastdate(args.days) + else: + args.begindate = tools.setBegindate(tools.toDatetime(args.begindate), tools.getLastdate(args.days)) + + if args.enddate is None: + args.enddate = datetime.now() + else: + args.enddate = tools.setEnddate(tools.toDatetime(args.enddate)) Events = tools.eventsListBuildFromArray('data') TotalEvents = tools.getNbitems(Events) diff --git a/examples/situational-awareness/tools.py b/examples/situational-awareness/tools.py index 57ee710..098d035 100644 --- a/examples/situational-awareness/tools.py +++ b/examples/situational-awareness/tools.py @@ -11,6 +11,15 @@ import pandas as pd from datetime import datetime from datetime import timedelta from dateutil.parser import parse +import sys + +################ Errors ################ + +class DateError(Exception): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) ################ Tools ################ @@ -52,8 +61,41 @@ def dateInRange(datetimeTested, begin=None, end=None): return begin <= datetimeTested <= end def toDatetime(date): - temp = date.split('-') - return datetime(int(temp[0]), int(temp[1]), int(temp[2])) + return parse(date) + +def checkDateConsistancy(begindate, enddate, lastdate): + try: + if begindate is not None and enddate is not None: + if begindate > enddate: + raise DateError('begindate (' + begindate + ') cannot be after enddate (' + enddate + ')') + except DateError as e: + print('DateError: ' + e.value) + sys.exit(1) + + try: + if enddate is not None: + if toDatetime(enddate) < lastdate: + raise DateError('enddate (' + enddate + ') cannot be before lastdate (' + str(lastdate) + ')' ) + except DateError as e: + print('DateError: ' + e.value) + sys.exit(1) + + try: + if begindate is not None: + if toDatetime(begindate) > datetime.now(): + raise DateError('begindate (' + begindate + ') cannot be after today (' + str(datetime.now().date()) + ')') + except DateError as e: + print('DateError: ' + e.value) + sys.exit(1) + +def setBegindate(begindate, lastdate): + return max(begindate, lastdate) + +def setEnddate(enddate): + return min(enddate, datetime.now()) + +def getLastdate(last): + return (datetime.now() - timedelta(days=int(last))).replace(hour=0, minute=0, second=0, microsecond=0) ################ Formatting ################ From f8dbcde607ffddde09455f557257b0693ffd573c Mon Sep 17 00:00:00 2001 From: Deborah Servili Date: Tue, 26 Jul 2016 11:09:00 +0200 Subject: [PATCH 072/223] Update README.md --- examples/situational-awareness/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/situational-awareness/README.md b/examples/situational-awareness/README.md index 58a1381..86f06bd 100644 --- a/examples/situational-awareness/README.md +++ b/examples/situational-awareness/README.md @@ -11,6 +11,7 @@ * _enddate_ is the end of the studied period. If it is earlier than _begindate_, an error will be raised. * tag\_search.py allows research for multiple tags is possible by separating each tag by the | symbol. * Partial research is also possible with tag\_search.py. For instance, search for "ransom" will also return tags containin "ransomware". + :warning: These scripts are not time optimised ## Requierements From 24d131aa32f15f85359996003f3f319985f50193 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 26 Jul 2016 16:35:46 +0200 Subject: [PATCH 073/223] Initial refactoring, PEP8 and cleanup --- .../attribute_treemap.py | 36 +--- examples/situational-awareness/style.css | 12 +- examples/situational-awareness/tag_search.py | 12 +- examples/situational-awareness/tags_count.py | 14 +- .../test_attribute_treemap.html | 4 +- examples/situational-awareness/tools.py | 169 ++++++++---------- 6 files changed, 108 insertions(+), 139 deletions(-) mode change 100644 => 100755 examples/situational-awareness/attribute_treemap.py diff --git a/examples/situational-awareness/attribute_treemap.py b/examples/situational-awareness/attribute_treemap.py old mode 100644 new mode 100755 index 1e7ef63..0536590 --- a/examples/situational-awareness/attribute_treemap.py +++ b/examples/situational-awareness/attribute_treemap.py @@ -4,45 +4,25 @@ from pymisp import PyMISP from keys import misp_url, misp_key, misp_verifycert import argparse -import os -import json import tools -def init(url, key): - return PyMISP(url, key, misp_verifycert, 'json') - -########## fetch data ########## - -def searchall(m, search, url): - result = m.search_all(search) - with open('data', 'w') as f: - f.write(json.dumps(result)) - -def download_last(m, last): - result = m.download_last(last) - with open('data', 'w') as f: - f.write(json.dumps(result)) - - if __name__ == '__main__': parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py of searchall.py) and create a treemap epresenting the distribution of attributes in this sample.') - parser.add_argument("-f", "--function", required=True, help="The parameter can be either set to \"last\" or \"searchall\". If the parameter is not valid, \"last\" will be the default setting.") - parser.add_argument("-a", "--argument", required=True, help="if function is \"last\", time can be defined in days, hours, minutes (for example 5d or 12h or 30m). Otherwise, this argument is the string to search") + parser.add_argument("-f", "--function", required=True, help='The parameter can be either set to "last" or "searchall". If the parameter is not valid, "last" will be the default setting.') + parser.add_argument("-a", "--argument", required=True, help='if function is "last", time can be defined in days, hours, minutes (for example 5d or 12h or 30m). Otherwise, this argument is the string to search') args = parser.parse_args() - misp = init(misp_url, misp_key) + misp = PyMISP(misp_url, misp_key, misp_verifycert, 'json') if args.function == "searchall": - searchall(misp, args.argument, misp_url) + result = misp.search_all(args.argument) else: - download_last(misp, args.argument) + result = misp.download_last(args.argument) - Events = tools.eventsListBuildFromArray('data') - - Attributes = tools.attributesListBuild(Events) - temp = tools.getNbAttributePerEventCategoryType(Attributes) + events = tools.eventsListBuildFromArray(result) + attributes = tools.attributesListBuild(events) + temp = tools.getNbAttributePerEventCategoryType(attributes) temp = temp.groupby(level=['category', 'type']).sum() tools.createTreemap(temp, 'Attributes Distribution', 'attribute_treemap.svg', 'attribute_table.html') - diff --git a/examples/situational-awareness/style.css b/examples/situational-awareness/style.css index 8c5313b..5afdf7f 100644 --- a/examples/situational-awareness/style.css +++ b/examples/situational-awareness/style.css @@ -1,4 +1,4 @@ -body +body { /*font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;*/ font-family: Consolas, "Liberation Mono", Menlo, Courier, monospace; @@ -20,27 +20,27 @@ table border: 1px solid #cbcbcb; } -tbody +tbody { font-size:12px; } -table td +table td { border-left: 1px solid #cbcbcb; border-width: 0 0 0 1px; - width: 150px; + width: 150px; margin: 0; padding: 0.5em 1em; } -table tr:nth-child(2n-1) td +table tr:nth-child(2n-1) td { background-color: #f2f2f2; } -table tr td:first-child +table tr td:first-child { font-weight: bold; } diff --git a/examples/situational-awareness/tag_search.py b/examples/situational-awareness/tag_search.py index d695a00..5a7c648 100644 --- a/examples/situational-awareness/tag_search.py +++ b/examples/situational-awareness/tag_search.py @@ -8,10 +8,12 @@ import argparse import json import tools + def init(url, key): return PyMISP(url, key, misp_verifycert, 'json') -########## fetch data ########## +# ######### fetch data ########## + def download_last(m, last): result = m.download_last(last) @@ -62,8 +64,8 @@ if __name__ == '__main__': else: text = text + str(args.enddate.date()) - print '\n========================================================' - print text - print 'During the studied pediod, ' + str(TotalPeriodTags) + ' events out of ' + str(TotalPeriodEvents) + ' contains at least one tag with ' + args.tag + '.' + print('\n========================================================') + print(text) + print('During the studied pediod, ' + str(TotalPeriodTags) + ' events out of ' + str(TotalPeriodEvents) + ' contains at least one tag with ' + args.tag + '.') if TotalPeriodEvents != 0: - print 'It represents ' + str(round(100*TotalPeriodTags/TotalPeriodEvents, 3)) + '% of the events in this period.' + print('It represents {}% of the events in this period.'.format(round(100 * TotalPeriodTags / TotalPeriodEvents, 3))) diff --git a/examples/situational-awareness/tags_count.py b/examples/situational-awareness/tags_count.py index 58e6194..8e9ce29 100644 --- a/examples/situational-awareness/tags_count.py +++ b/examples/situational-awareness/tags_count.py @@ -8,10 +8,12 @@ import argparse import json import tools + def init(url, key): return PyMISP(url, key, misp_verifycert, 'json') -########## fetch data ########## +# ######### fetch data ########## + def download_last(m, last): result = m.download_last(last) @@ -21,7 +23,7 @@ def download_last(m, last): if __name__ == '__main__': parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py) and give the repartition of tags in this sample.') parser.add_argument("-d", "--days", type=int, help="number of days before today to search. If not define, default value is 7") - parser.add_argument("-b", "--begindate", help="The research will look for tags attached to events posted at or after the given startdate (format: yyyy-mm-dd): If no date is given, default time is epoch time (1970-1-1)") + parser.add_argument("-b", "--begindate", default='1970-01-01', help="The research will look for tags attached to events posted at or after the given startdate (format: yyyy-mm-dd): If no date is given, default time is epoch time (1970-1-1)") parser.add_argument("-e", "--enddate", help="The research will look for tags attached to events posted at or before the given enddate (format: yyyy-mm-dd): If no date is given, default time is now()") args = parser.parse_args() @@ -30,7 +32,7 @@ if __name__ == '__main__': if args.days is None: args.days = 7 - download_last(misp, str(args.days) + 'd') + download_last(misp, '{}d'.format(args.days)) tools.checkDateConsistancy(args.begindate, args.enddate, tools.getLastdate(args.days)) @@ -67,6 +69,6 @@ if __name__ == '__main__': else: text = text + str(args.enddate.date()) - print '\n========================================================' - print text - print result + print('\n========================================================') + print(text) + print(result) diff --git a/examples/situational-awareness/test_attribute_treemap.html b/examples/situational-awareness/test_attribute_treemap.html index d6e8fc4..0bc9c72 100644 --- a/examples/situational-awareness/test_attribute_treemap.html +++ b/examples/situational-awareness/test_attribute_treemap.html @@ -15,11 +15,11 @@ - +
diff --git a/examples/situational-awareness/tools.py b/examples/situational-awareness/tools.py index 098d035..232ce8e 100644 --- a/examples/situational-awareness/tools.py +++ b/examples/situational-awareness/tools.py @@ -1,107 +1,97 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import json from json import JSONDecoder -import math import random import pygal from pygal.style import Style -import pandas as pd +import pandas from datetime import datetime from datetime import timedelta from dateutil.parser import parse -import sys -################ Errors ################ +# ############### Errors ################ + class DateError(Exception): def __init__(self, value): self.value = value + def __str__(self): return repr(self.value) -################ Tools ################ +# ############### Tools ################ + def buildDoubleIndex(index1, index2, datatype): it = -1 newindex1 = [] for index in index2: if index == 0: - it+=1 + it += 1 newindex1.append(index1[it]) - arrays = [newindex1, index2] + arrays = [newindex1, index2] tuples = list(zip(*arrays)) - return pd.MultiIndex.from_tuples(tuples, names=['event', datatype]) + return pandas.MultiIndex.from_tuples(tuples, names=['event', datatype]) + def buildNewColumn(index2, column): it = -1 newcolumn = [] for index in index2: if index == 0: - it+=1 + it += 1 newcolumn.append(column[it]) return newcolumn + def dateInRange(datetimeTested, begin=None, end=None): - if begin == None: - begin = datetime(1970,1,1) - if end == None: + if begin is None: + begin = datetime(1970, 1, 1) + if end is None: end = datetime.now() return begin <= datetimeTested <= end + def addColumn(dataframe, columnList, columnName): - dataframe.loc[:, columnName] = pd.Series(columnList, index=dataframe.index) + dataframe.loc[:, columnName] = pandas.Series(columnList, index=dataframe.index) -def dateInRange(datetimeTested, begin=None, end=None): - if begin == None: - begin = datetime(1970,1,1) - if end == None: - end = datetime.now() - return begin <= datetimeTested <= end def toDatetime(date): return parse(date) + def checkDateConsistancy(begindate, enddate, lastdate): - try: - if begindate is not None and enddate is not None: - if begindate > enddate: - raise DateError('begindate (' + begindate + ') cannot be after enddate (' + enddate + ')') - except DateError as e: - print('DateError: ' + e.value) - sys.exit(1) + if begindate is not None and enddate is not None: + if begindate > enddate: + raise DateError('begindate ({}) cannot be after enddate ({})'.format(begindate, enddate)) - try: - if enddate is not None: - if toDatetime(enddate) < lastdate: - raise DateError('enddate (' + enddate + ') cannot be before lastdate (' + str(lastdate) + ')' ) - except DateError as e: - print('DateError: ' + e.value) - sys.exit(1) + if enddate is not None: + if toDatetime(enddate) < lastdate: + raise DateError('enddate ({}) cannot be before lastdate ({})'.format(enddate, lastdate)) + + if begindate is not None: + if toDatetime(begindate) > datetime.now(): + raise DateError('begindate ({}) cannot be after today ({})'.format(begindate, datetime.now().date())) - try: - if begindate is not None: - if toDatetime(begindate) > datetime.now(): - raise DateError('begindate (' + begindate + ') cannot be after today (' + str(datetime.now().date()) + ')') - except DateError as e: - print('DateError: ' + e.value) - sys.exit(1) def setBegindate(begindate, lastdate): return max(begindate, lastdate) + def setEnddate(enddate): return min(enddate, datetime.now()) + def getLastdate(last): return (datetime.now() - timedelta(days=int(last))).replace(hour=0, minute=0, second=0, microsecond=0) -################ Formatting ################ +# ############### Formatting ################ + def eventsListBuildFromList(filename): with open(filename, 'r') as myfile: - s=myfile.read().replace('\n', '') + s = myfile.read().replace('\n', '') decoder = JSONDecoder() s_len = len(s) Events = [] @@ -111,66 +101,57 @@ def eventsListBuildFromList(filename): Events.append(Event) data = [] for e in Events: - data.append(pd.DataFrame.from_dict(e, orient='index')) - Events = pd.concat(data) + data.append(pandas.DataFrame.from_dict(e, orient='index')) + Events = pandas.concat(data) for it in range(Events['attribute_count'].size): - if Events['attribute_count'][it] == None: - Events['attribute_count'][it]='0' + if Events['attribute_count'][it] is None: + Events['attribute_count'][it] = '0' else: - Events['attribute_count'][it]=int(Events['attribute_count'][it]) + Events['attribute_count'][it] = int(Events['attribute_count'][it]) Events = Events.set_index('id') return Events -def eventsListBuildFromArray(filename): + +def eventsListBuildFromArray(jdata): ''' returns a structure listing all primary events in the sample ''' - jdata = json.load(open(filename)) - jdata = jdata['response'] - Events = [] - for e in jdata: - Events.append(e) - data = [] - for e in Events: - data.append(pd.DataFrame.from_dict(e, orient='index')) - Events = pd.concat(data) - for it in range(Events['attribute_count'].size): - if Events['attribute_count'][it] == None or (isinstance(Events['attribute_count'][it], float) and math.isnan(Events['attribute_count'][it])): - Events['attribute_count'][it]='0' - else: - Events['attribute_count'][it]=int(Events['attribute_count'][it]) - Events = Events.set_index('id') - return Events + data = [pandas.DataFrame.from_dict(e, orient='index') for e in jdata['response']] + events = pandas.concat(data) + events = events.set_index(['id']) + return events + + +def attributesListBuild(events): + attributes = [pandas.DataFrame(attribute) for attribute in events['Attribute']] + return pandas.concat(attributes) -def attributesListBuild(Events): - Attributes = [] - for Attribute in Events['Attribute']: - Attributes.append(pd.DataFrame(Attribute)) - return pd.concat(Attributes) def tagsListBuild(Events): Tags = [] for Tag in Events['Tag']: if type(Tag) is not list: continue - Tags.append(pd.DataFrame(Tag)) - Tags = pd.concat(Tags) + Tags.append(pandas.DataFrame(Tag)) + Tags = pandas.concat(Tags) columnDate = buildNewColumn(Tags.index, Events['date']) addColumn(Tags, columnDate, 'date') index = buildDoubleIndex(Events.index, Tags.index, 'tag') Tags = Tags.set_index(index) return Tags + def selectInRange(Events, begin=None, end=None): inRange = [] for i, Event in Events.iterrows(): if dateInRange(parse(Event['date']), begin, end): inRange.append(Event.tolist()) - inRange = pd.DataFrame(inRange) + inRange = pandas.DataFrame(inRange) temp = Events.columns.tolist() inRange.columns = temp return inRange + def isTagIn(dataframe, tag): temp = dataframe[dataframe['name'].str.contains(tag)].index.tolist() index = [] @@ -179,35 +160,39 @@ def isTagIn(dataframe, tag): index.append(temp[i][0]) return index -################ Basic Stats ################ +# ############### Basic Stats ################ + def getNbitems(dataframe): return len(dataframe.index) -def getNbAttributePerEventCategoryType(Attributes): - return Attributes.groupby(['event_id', 'category', 'type']).count()['id'] + +def getNbAttributePerEventCategoryType(attributes): + return attributes.groupby(['event_id', 'category', 'type']).count()['id'] + def getNbOccurenceTags(Tags): return Tags.groupby('name').count()['id'] -################ Charts ################ +# ############### Charts ################ + def createStyle(indexlevels): colorsList = [] for i in range(len(indexlevels[0])): colorsList.append("#%06X" % random.randint(0, 0xFFFFFF)) - style = Style( - background='transparent', - plot_background='#FFFFFF', - foreground='#111111', - foreground_strong='#111111', - foreground_subtle='#111111', - opacity='.6', - opacity_hover='.9', - transition='400ms ease-in', - colors=tuple(colorsList)) + style = Style(background='transparent', + plot_background='#FFFFFF', + foreground='#111111', + foreground_strong='#111111', + foreground_subtle='#111111', + opacity='.6', + opacity_hover='.9', + transition='400ms ease-in', + colors=tuple(colorsList)) return style, colorsList + def createLabelsTreemap(indexlevels, indexlabels): categories_levels = indexlevels[0] cat = 0 @@ -230,7 +215,7 @@ def createLabelsTreemap(indexlevels, indexlabels): def createTable(data, title, tablename, colorsList): - if tablename == None: + if tablename is None: target = open('attribute_table.html', 'w') else: target = open(tablename, 'w') @@ -241,7 +226,7 @@ def createTable(data, title, tablename, colorsList): for i in range(len(categories)): table = pygal.Treemap(pretty_print=True) - target.write('\n

' + categories[i] + '

\n') + target.write('\n

{}

\n'.format(colorsList[i], categories[i])) for typ in types[i]: table.add(typ, data[it]) it += 1 @@ -250,9 +235,9 @@ def createTable(data, title, tablename, colorsList): target.close() -def createTreemap(data, title, treename = 'attribute_treemap.svg', tablename = 'attribute_table.html'): +def createTreemap(data, title, treename='attribute_treemap.svg', tablename='attribute_table.html'): style, colorsList = createStyle(data.index.levels) - treemap = pygal.Treemap(pretty_print=True, legend_at_bottom=True, style = style) + treemap = pygal.Treemap(pretty_print=True, legend_at_bottom=True, style=style) treemap.title = title treemap.print_values = True treemap.print_labels = True @@ -271,7 +256,7 @@ def createTreemap(data, title, treename = 'attribute_treemap.svg', tablename = ' treemap.add(categories[i], types_labels) createTable(data, 'Attribute Distribution', tablename, colorsList) - if treename == None: + if treename is None: treemap.render_to_file('attribute_treemap.svg') else: treemap.render_to_file(treename) From d31ec7a73c8ff8bde29d21cc830011e79852c3db Mon Sep 17 00:00:00 2001 From: Jessy Campos Date: Tue, 26 Jul 2016 13:13:29 -0400 Subject: [PATCH 074/223] Add a method to add a textual detection name under the 'Antivirus detection' category --- pymisp/api.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pymisp/api.py b/pymisp/api.py index 3ae0d12..266fb35 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -451,6 +451,11 @@ class PyMISP(object): attributes.append(self._prepare_full_attribute(category, 'link', link, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) + def add_detection_name(self, event, name, category='Antivirus detection', to_ids=False, comment=None, distribution=None, proposal=False): + attributes = [] + attributes.append(self._prepare_full_attribute(category, 'text', name, to_ids, comment, distribution)) + return self._send_attributes(event, attributes, proposal) + def add_filename(self, event, filename, category='Artifacts dropped', to_ids=False, comment=None, distribution=None, proposal=False): attributes = [] attributes.append(self._prepare_full_attribute(category, 'filename', filename, to_ids, comment, distribution)) From 90b772d9389f23fa621707378ebd740e35597414 Mon Sep 17 00:00:00 2001 From: Kenneth Adam Miller Date: Wed, 27 Jul 2016 07:30:46 -0400 Subject: [PATCH 075/223] Caught exception on python3.4 where base64encode returns bytes and not str, and bytes are not json encodable. This caused a failure in upload_sample --- pymisp/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/api.py b/pymisp/api.py index 266fb35..b5560ce 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -677,7 +677,7 @@ class PyMISP(object): def _encode_file_to_upload(self, path): with open(path, 'rb') as f: - return base64.b64encode(f.read()) + return str(base64.b64encode(f.read())) def upload_sample(self, filename, filepath, event_id, distribution, to_ids, category, comment, info, analysis, threat_level_id): From 90bb9f3ba43e82848933ead1816e1d564671386d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 27 Jul 2016 14:48:13 +0200 Subject: [PATCH 076/223] Major refactoring of the SVG generator --- examples/situational-awareness/tools.py | 98 ++++++++----------------- 1 file changed, 30 insertions(+), 68 deletions(-) diff --git a/examples/situational-awareness/tools.py b/examples/situational-awareness/tools.py index 232ce8e..8b3a8b7 100644 --- a/examples/situational-awareness/tools.py +++ b/examples/situational-awareness/tools.py @@ -177,10 +177,31 @@ def getNbOccurenceTags(Tags): # ############### Charts ################ -def createStyle(indexlevels): - colorsList = [] - for i in range(len(indexlevels[0])): - colorsList.append("#%06X" % random.randint(0, 0xFFFFFF)) +def createTable(colors, categ_types_hash, tablename='attribute_table.html'): + with open(tablename, 'w') as target: + target.write('\n\n\n\n\n') + for categ_name, types in categ_types_hash.items(): + table = pygal.Treemap(pretty_print=True) + target.write('\n

{}

\n'.format(colors[categ_name], categ_name)) + for d in types: + table.add(d['label'], d['value']) + target.write(table.render_table(transpose=True)) + target.write('\n\n') + + +def createTreemap(data, title, treename='attribute_treemap.svg', tablename='attribute_table.html'): + labels_categ = data.index.labels[0] + labels_types = data.index.labels[1] + names_categ = data.index.levels[0] + names_types = data.index.levels[1] + categ_types_hash = {} + for categ_id, type_val, total in zip(labels_categ, labels_types, data): + if not categ_types_hash.get(names_categ[categ_id]): + categ_types_hash[names_categ[categ_id]] = [] + dict_to_print = {'label': names_types[type_val], 'value': total} + categ_types_hash[names_categ[categ_id]].append(dict_to_print) + + colors = {categ: "#%06X" % random.randint(0, 0xFFFFFF) for categ in categ_types_hash.keys()} style = Style(background='transparent', plot_background='#FFFFFF', foreground='#111111', @@ -189,74 +210,15 @@ def createStyle(indexlevels): opacity='.6', opacity_hover='.9', transition='400ms ease-in', - colors=tuple(colorsList)) - return style, colorsList + colors=tuple(colors.values())) - -def createLabelsTreemap(indexlevels, indexlabels): - categories_levels = indexlevels[0] - cat = 0 - types = [] - cattypes = [] - categories_labels = indexlabels[0] - types_levels = indexlevels[1] - types_labels = indexlabels[1] - - for it in range(len(indexlabels[0])): - if categories_labels[it] != cat: - cattypes.append(types) - types = [] - cat += 1 - - types.append(types_levels[types_labels[it]]) - cattypes.append(types) - - return categories_levels, cattypes - - -def createTable(data, title, tablename, colorsList): - if tablename is None: - target = open('attribute_table.html', 'w') - else: - target = open(tablename, 'w') - target.truncate() - target.write('\n\n\n\n\n') - categories, types = createLabelsTreemap(data.index.levels, data.index.labels) - it = 0 - - for i in range(len(categories)): - table = pygal.Treemap(pretty_print=True) - target.write('\n

{}

\n'.format(colorsList[i], categories[i])) - for typ in types[i]: - table.add(typ, data[it]) - it += 1 - target.write(table.render_table(transpose=True)) - target.write('\n\n') - target.close() - - -def createTreemap(data, title, treename='attribute_treemap.svg', tablename='attribute_table.html'): - style, colorsList = createStyle(data.index.levels) treemap = pygal.Treemap(pretty_print=True, legend_at_bottom=True, style=style) treemap.title = title treemap.print_values = True treemap.print_labels = True - categories, types = createLabelsTreemap(data.index.levels, data.index.labels) - it = 0 + for categ_name, types in categ_types_hash.items(): + treemap.add(categ_name, types) - for i in range(len(categories)): - types_labels = [] - for typ in types[i]: - tempdict = {} - tempdict['label'] = typ - tempdict['value'] = data[it] - types_labels.append(tempdict) - it += 1 - treemap.add(categories[i], types_labels) - - createTable(data, 'Attribute Distribution', tablename, colorsList) - if treename is None: - treemap.render_to_file('attribute_treemap.svg') - else: - treemap.render_to_file(treename) + createTable(colors, categ_types_hash) + treemap.render_to_file(treename) From f45490b02e74f84450249603fa6927d2c69a3661 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 28 Jul 2016 09:49:40 +0200 Subject: [PATCH 077/223] Add support for proxies in the library. Fix #48 --- pymisp/api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pymisp/api.py b/pymisp/api.py index 266fb35..34ce17a 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -93,7 +93,7 @@ class PyMISP(object): :param out_type: Type of object (json or xml) """ - def __init__(self, url, key, ssl=True, out_type='json', debug=False): + def __init__(self, url, key, ssl=True, out_type='json', debug=False, proxies=None): if not url: raise NoURL('Please provide the URL of your MISP instance.') if not key: @@ -102,6 +102,7 @@ class PyMISP(object): self.root_url = url self.key = key self.ssl = ssl + self.proxies = proxies self.out_type = out_type self.debug = debug @@ -134,6 +135,7 @@ class PyMISP(object): out = self.out_type session = requests.Session() session.verify = self.ssl + session.proxies = self.proxies session.headers.update( {'Authorization': self.key, 'Accept': 'application/' + out, From b8205f11a85e463398e96273aeeec6089118fdac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 28 Jul 2016 09:50:46 +0200 Subject: [PATCH 078/223] Fix PEP8 --- pymisp/api.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 34ce17a..be40010 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -225,18 +225,16 @@ class PyMISP(object): """ out_format = out_format.lower() if tags: - if isinstance(tags, list): - tags = "&&".join(tags) - + if isinstance(tags, list): + tags = "&&".join(tags) + session = self.__prepare_session(out_format) - url = urljoin(self.root_url, - "/events/stix/download/{}/{}/{}/{}/{}".format( - event_id, with_attachments, tags, from_date, to_date - )) + url = urljoin(self.root_url, "/events/stix/download/{}/{}/{}/{}/{}".format( + event_id, with_attachments, tags, from_date, to_date)) if self.debug: - print("Getting STIX event from {}".format(url)) + print("Getting STIX event from {}".format(url)) return session.get(url) - + def add_event(self, event, force_out=None): """ Add a new event From 93ef3595e5f1caea9fa504262a6f27f296820386 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 29 Jul 2016 13:25:26 +0200 Subject: [PATCH 079/223] Fix fetching method for tag_search and tags_count --- examples/situational-awareness/tag_search.py | 24 ++++++++------------ examples/situational-awareness/tags_count.py | 22 ++++-------------- 2 files changed, 13 insertions(+), 33 deletions(-) diff --git a/examples/situational-awareness/tag_search.py b/examples/situational-awareness/tag_search.py index 5a7c648..490d0ff 100644 --- a/examples/situational-awareness/tag_search.py +++ b/examples/situational-awareness/tag_search.py @@ -5,7 +5,6 @@ from pymisp import PyMISP from keys import misp_url, misp_key, misp_verifycert from datetime import datetime import argparse -import json import tools @@ -15,11 +14,6 @@ def init(url, key): # ######### fetch data ########## -def download_last(m, last): - result = m.download_last(last) - with open('data', 'w') as f: - f.write(json.dumps(result)) - if __name__ == '__main__': parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py) and give the number of occurrence of the given tag in this sample.') parser.add_argument("-t", "--tag", required=True, help="tag to search (search for multiple tags is possible by using |. example : \"osint|OSINT\")") @@ -33,7 +27,7 @@ if __name__ == '__main__': if args.days is None: args.days = 7 - download_last(misp, str(args.days) + 'd') + result = misp.download_last('{}d'.format(args.days)) tools.checkDateConsistancy(args.begindate, args.enddate, tools.getLastdate(args.days)) @@ -47,11 +41,11 @@ if __name__ == '__main__': else: args.enddate = tools.setEnddate(tools.toDatetime(args.enddate)) - Events = tools.selectInRange(tools.eventsListBuildFromArray('data'), begin=args.begindate, end=args.enddate) - TotalPeriodEvents = tools.getNbitems(Events) - Tags = tools.tagsListBuild(Events) - result = tools.isTagIn(Tags, args.tag) - TotalPeriodTags = len(result) + events = tools.selectInRange(tools.eventsListBuildFromArray(result), begin=args.begindate, end=args.enddate) + totalPeriodEvents = tools.getNbitems(events) + tags = tools.tagsListBuild(events) + result = tools.isTagIn(tags, args.tag) + totalPeriodTags = len(result) text = 'Studied pediod: from ' if args.begindate is None: @@ -66,6 +60,6 @@ if __name__ == '__main__': print('\n========================================================') print(text) - print('During the studied pediod, ' + str(TotalPeriodTags) + ' events out of ' + str(TotalPeriodEvents) + ' contains at least one tag with ' + args.tag + '.') - if TotalPeriodEvents != 0: - print('It represents {}% of the events in this period.'.format(round(100 * TotalPeriodTags / TotalPeriodEvents, 3))) + print('During the studied pediod, ' + str(totalPeriodTags) + ' events out of ' + str(totalPeriodEvents) + ' contains at least one tag with ' + args.tag + '.') + if totalPeriodEvents != 0: + print('It represents {}% of the events in this period.'.format(round(100 * totalPeriodTags / totalPeriodEvents, 3))) diff --git a/examples/situational-awareness/tags_count.py b/examples/situational-awareness/tags_count.py index 8e9ce29..f925574 100644 --- a/examples/situational-awareness/tags_count.py +++ b/examples/situational-awareness/tags_count.py @@ -5,7 +5,6 @@ from pymisp import PyMISP from keys import misp_url, misp_key, misp_verifycert from datetime import datetime import argparse -import json import tools @@ -15,11 +14,6 @@ def init(url, key): # ######### fetch data ########## -def download_last(m, last): - result = m.download_last(last) - with open('data', 'w') as f: - f.write(json.dumps(result)) - if __name__ == '__main__': parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py) and give the repartition of tags in this sample.') parser.add_argument("-d", "--days", type=int, help="number of days before today to search. If not define, default value is 7") @@ -32,7 +26,7 @@ if __name__ == '__main__': if args.days is None: args.days = 7 - download_last(misp, '{}d'.format(args.days)) + result = misp.download_last('{}d'.format(args.days)) tools.checkDateConsistancy(args.begindate, args.enddate, tools.getLastdate(args.days)) @@ -46,17 +40,9 @@ if __name__ == '__main__': else: args.enddate = tools.setEnddate(tools.toDatetime(args.enddate)) - Events = tools.eventsListBuildFromArray('data') - TotalEvents = tools.getNbitems(Events) - Tags = tools.tagsListBuild(Events) - result = tools.getNbOccurenceTags(Tags) - TotalTags = tools.getNbitems(Tags) - - Events = tools.selectInRange(Events, begin=args.begindate, end=args.enddate) - TotalPeriodEvents = tools.getNbitems(Events) - Tags = tools.tagsListBuild(Events) - result = tools.getNbOccurenceTags(Tags) - TotalPeriodTags = tools.getNbitems(Tags) + events = tools.selectInRange(tools.eventsListBuildFromArray(result), begin=args.begindate, end=args.enddate) + tags = tools.tagsListBuild(events) + result = tools.getNbOccurenceTags(tags) text = 'Studied pediod: from ' if args.begindate is None: From 5937ef9e9b453749c0988d28423d90efd20c6c50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 2 Aug 2016 15:17:42 +0200 Subject: [PATCH 080/223] Version bump --- examples/get.py | 13 ++++++++++--- examples/situational-awareness/tools.py | 2 +- pymisp/__init__.py | 2 +- setup.py | 5 ++++- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/examples/get.py b/examples/get.py index 5da2d4e..71bfa97 100755 --- a/examples/get.py +++ b/examples/get.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- from pymisp import PyMISP -from keys import misp_url, misp_key,misp_verifycert +from keys import misp_url, misp_key, misp_verifycert import argparse import os import json @@ -10,9 +10,16 @@ import json # Usage for pipe masters: ./last.py -l 5h | jq . +proxies = { + 'http': 'http://127.0.0.1:8123', + 'https': 'http://127.0.0.1:8123', +} + +proxies = None + def init(url, key): - return PyMISP(url, key, misp_verifycert, 'json') + return PyMISP(url, key, misp_verifycert, 'json', proxies=proxies) def get_event(m, event, out=None): @@ -24,8 +31,8 @@ def get_event(m, event, out=None): with open(out, 'w') as f: f.write(json.dumps(r) + '\n') - if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Get an event from a MISP instance.') parser.add_argument("-e", "--event", required=True, help="Event ID to get.") parser.add_argument("-o", "--output", help="Output file") diff --git a/examples/situational-awareness/tools.py b/examples/situational-awareness/tools.py index 8b3a8b7..6cff510 100644 --- a/examples/situational-awareness/tools.py +++ b/examples/situational-awareness/tools.py @@ -212,7 +212,7 @@ def createTreemap(data, title, treename='attribute_treemap.svg', tablename='attr transition='400ms ease-in', colors=tuple(colors.values())) - treemap = pygal.Treemap(pretty_print=True, legend_at_bottom=True, style=style) + treemap = pygal.Treemap(pretty_print=True, legend_at_bottom=True, style=style, explicit_size=True, width=2048, height=2048) treemap.title = title treemap.print_values = True treemap.print_labels = True diff --git a/pymisp/__init__.py b/pymisp/__init__.py index c6a29ba..aea8a6c 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,3 +1,3 @@ -__version__ = '2.4.48.2' +__version__ = '2.4.49' from .api import PyMISP, PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey diff --git a/setup.py b/setup.py index 0e8f3e9..5cab2ca 100644 --- a/setup.py +++ b/setup.py @@ -17,9 +17,12 @@ setup( 'License :: OSI Approved :: BSD License', 'Development Status :: 5 - Production/Stable', 'Environment :: Console', + 'Operating System :: POSIX :: Linux', 'Intended Audience :: Science/Research', 'Intended Audience :: Telecommunications Industry', - 'Programming Language :: Python', + 'Intended Audience :: Information Technology', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', 'Topic :: Security', 'Topic :: Internet', ], From cb25177a4687060ff69bfc7cdf54a4cf96da4687 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 4 Aug 2016 13:21:28 +0200 Subject: [PATCH 081/223] Add option to search function to only return attributes instead of events --- pymisp/api.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 780fd4a..b92cdb3 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -90,7 +90,9 @@ class PyMISP(object): of the certificate. Or a CA_BUNDLE in case of self signed certiifcate (the concatenation of all the *.crt of the chain) - :param out_type: Type of object (json or xml) + :param out_type: Type of object (json or xml) NOTE: XML is deprecated. + :param debug: print all the messages received from the server + :param proxies: Proxy dict as describes here: http://docs.python-requests.org/en/master/user/advanced/#proxies """ def __init__(self, url, key, ssl=True, out_type='json', debug=False, proxies=None): @@ -753,10 +755,12 @@ class PyMISP(object): # ######## REST Search ######### # ############################## - def __query(self, session, path, query): + def __query(self, session, path, query, controller='events'): if query.get('error') is not None: return query - url = urljoin(self.root_url, 'events/{}'.format(path.lstrip('/'))) + if controller not in ['events', 'attributes']: + raise Exception('Invalid controller. Can only be {}'.format(', '.join(['events', 'attributes']))) + url = urljoin(self.root_url, '{}/{}'.format(controller, path.lstrip('/'))) query = {'request': query} response = session.post(url, data=json.dumps(query)) return self._check_response(response) @@ -837,7 +841,7 @@ class PyMISP(object): def search(self, values=None, not_values=None, type_attribute=None, category=None, org=None, tags=None, not_tags=None, date_from=None, - date_to=None, last=None): + date_to=None, last=None, controller='events'): """ Search via the Rest API @@ -880,7 +884,7 @@ class PyMISP(object): query['last'] = last session = self.__prepare_session('json') - return self.__query(session, 'restSearch/download', query) + return self.__query(session, 'restSearch/download', query, controller) def get_attachement(self, event_id): """ From 86758cce19d63a6520db50f88359cb85547191ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 5 Aug 2016 11:13:26 +0200 Subject: [PATCH 082/223] Properly handle errors while fetching the types. Fix #53 --- pymisp/api.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pymisp/api.py b/pymisp/api.py index b92cdb3..9389bcc 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -115,7 +115,11 @@ class PyMISP(object): raise PyMISPError('Unable to connect to MISP ({}). Please make sure the API key and the URL are correct (http/https is required): {}'.format(self.root_url, e)) session = self.__prepare_session(out_type) - self.describe_types = session.get(urljoin(self.root_url, 'attributes/describeTypes.json')).json() + response = session.get(urljoin(self.root_url, 'attributes/describeTypes.json')) + self.describe_types = self._check_response(response) + if self.describe_types.get('error'): + for e in self.describe_types.get('error'): + raise PyMISPError('Failed: {}'.format(e)) self.categories = self.describe_types['result']['categories'] self.types = self.describe_types['result']['types'] From af4476096722820f4f91f048d4a82f6681ada812 Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Tue, 9 Aug 2016 13:31:44 +0200 Subject: [PATCH 083/223] Set default distribution for attributes to inherit --- pymisp/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/api.py b/pymisp/api.py index 9389bcc..ab9a75d 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -415,7 +415,7 @@ class PyMISP(object): event = self._prepare_update(event) for a in attributes: if a.get('distribution') is None: - a['distribution'] = event['Event']['distribution'] + a['distribution'] = 5 event['Event']['Attribute'] = attributes response = self.update_event(event['Event']['id'], event, 'json') return self._check_response(response) From 2a0d6566eec52262b4e351a513904365378b5705 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 9 Aug 2016 13:58:54 +0200 Subject: [PATCH 084/223] change: remove XML output, all functions return a Python dictionary. --- pymisp/api.py | 164 +++++++++++++++++++++----------------------------- 1 file changed, 68 insertions(+), 96 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index ab9a75d..a9979fe 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -90,7 +90,7 @@ class PyMISP(object): of the certificate. Or a CA_BUNDLE in case of self signed certiifcate (the concatenation of all the *.crt of the chain) - :param out_type: Type of object (json or xml) NOTE: XML is deprecated. + :param out_type: Type of object (json) NOTE: XML output isn't supported anymore, keeping the flag for compatibility reasons. :param debug: print all the messages received from the server :param proxies: Proxy dict as describes here: http://docs.python-requests.org/en/master/user/advanced/#proxies """ @@ -105,7 +105,8 @@ class PyMISP(object): self.key = key self.ssl = ssl self.proxies = proxies - self.out_type = out_type + if out_type != 'json': + raise PyMISPError('The only output type supported by PyMISP is JSON. If you still rely on XML, use PyMISP v2.4.49') self.debug = debug try: @@ -114,7 +115,7 @@ class PyMISP(object): except Exception as e: raise PyMISPError('Unable to connect to MISP ({}). Please make sure the API key and the URL are correct (http/https is required): {}'.format(self.root_url, e)) - session = self.__prepare_session(out_type) + session = self.__prepare_session() response = session.get(urljoin(self.root_url, 'attributes/describeTypes.json')) self.describe_types = self._check_response(response) if self.describe_types.get('error'): @@ -125,27 +126,19 @@ class PyMISP(object): self.types = self.describe_types['result']['types'] self.category_type_mapping = self.describe_types['result']['category_type_mappings'] - def __prepare_session(self, force_out=None): + def __prepare_session(self): """ Prepare the headers of the session - - :param force_out: force the type of the expect output - (overwrite the constructor) - """ if not HAVE_REQUESTS: raise MissingDependency('Missing dependency, install requests (`pip install requests`)') - if force_out is not None: - out = force_out - else: - out = self.out_type session = requests.Session() session.verify = self.ssl session.proxies = self.proxies session.headers.update( {'Authorization': self.key, - 'Accept': 'application/' + out, - 'content-type': 'application/' + out}) + 'Accept': 'application/json', + 'content-type': 'application/json'}) return session def flatten_error_messages(self, response): @@ -200,94 +193,93 @@ class PyMISP(object): # ############### Simple REST API ################ # ################################################ - def get_index(self, force_out=None, filters=None): + def get_index(self, filters=None): """ Return the index. Warning, there's a limit on the number of results """ - session = self.__prepare_session(force_out) + session = self.__prepare_session() url = urljoin(self.root_url, 'events/index') if filters is not None: filters = json.dumps(filters) - print(filters) - return session.post(url, data=filters) + response = session.post(url, data=filters) else: - return session.get(url) + response = session.get(url) + return self._check_response(response) - def get_event(self, event_id, force_out=None): + def get_event(self, event_id): """ Get an event :param event_id: Event id to get """ - session = self.__prepare_session(force_out) + session = self.__prepare_session() url = urljoin(self.root_url, 'events/{}'.format(event_id)) - return session.get(url) + response = session.get(url) + return self._check_response(response) - def get_stix_event(self, event_id=None, out_format="json", with_attachments=False, from_date=False, to_date=False, tags=False): + def get_stix_event(self, event_id=None, with_attachments=False, from_date=False, to_date=False, tags=False): """ Get an event/events in STIX format """ - out_format = out_format.lower() if tags: if isinstance(tags, list): tags = "&&".join(tags) - session = self.__prepare_session(out_format) + session = self.__prepare_session() url = urljoin(self.root_url, "/events/stix/download/{}/{}/{}/{}/{}".format( event_id, with_attachments, tags, from_date, to_date)) if self.debug: print("Getting STIX event from {}".format(url)) - return session.get(url) + response = session.get(url) + return self._check_response(response) - def add_event(self, event, force_out=None): + def add_event(self, event): """ Add a new event :param event: Event as JSON object / string or XML to add """ - session = self.__prepare_session(force_out) + session = self.__prepare_session() url = urljoin(self.root_url, 'events') - if self.out_type == 'json': - if isinstance(event, basestring): - return session.post(url, data=event) - else: - return session.post(url, data=json.dumps(event)) + if isinstance(event, basestring): + response = session.post(url, data=event) else: - return session.post(url, data=event) + response = session.post(url, data=json.dumps(event)) + return self._check_response(response) - def update_event(self, event_id, event, force_out=None): + def update_event(self, event_id, event): """ Update an event :param event_id: Event id to update :param event: Event as JSON object / string or XML to add """ - session = self.__prepare_session(force_out) + session = self.__prepare_session() url = urljoin(self.root_url, 'events/{}'.format(event_id)) - if self.out_type == 'json': - if isinstance(event, basestring): - return session.post(url, data=event) - else: - return session.post(url, data=json.dumps(event)) + if isinstance(event, basestring): + response = session.post(url, data=event) else: - return session.post(url, data=event) + response = session.post(url, data=json.dumps(event)) + return self._check_response(response) - def delete_event(self, event_id, force_out=None): + def delete_event(self, event_id): """ Delete an event :param event_id: Event id to delete """ - session = self.__prepare_session(force_out) + session = self.__prepare_session() url = urljoin(self.root_url, 'events/{}'.format(event_id)) - return session.delete(url) + response = session.delete(url) + return self._check_response(response) - def delete_attribute(self, attribute_id, force_out=None): - session = self.__prepare_session(force_out) + def delete_attribute(self, attribute_id): + session = self.__prepare_session() url = urljoin(self.root_url, 'attributes/{}'.format(attribute_id)) - return session.delete(url) + response = session.delete(url) + return self._check_response(response) # ############################################## # ######### Event handling (Json only) ######### @@ -389,21 +381,18 @@ class PyMISP(object): session = self.__prepare_session('json') to_post = {'request': {'Event': {'id': event['Event']['id'], 'tag': tag}}} response = session.post(urljoin(self.root_url, 'events/addTag'), data=json.dumps(to_post)) - return self._check_response(response) def remove_tag(self, event, tag): session = self.__prepare_session('json') to_post = {'request': {'Event': {'id': event['Event']['id'], 'tag': tag}}} response = session.post(urljoin(self.root_url, 'events/removeTag'), data=json.dumps(to_post)) - return self._check_response(response) def change_threat_level(self, event, threat_level_id): event['Event']['threat_level_id'] = threat_level_id self._prepare_update(event) response = self.update_event(event['Event']['id'], event) - return self._check_response(response) # ##### File attributes ##### @@ -899,7 +888,8 @@ class PyMISP(object): """ attach = urljoin(self.root_url, 'attributes/downloadAttachment/download/{}'.format(event_id)) session = self.__prepare_session('json') - return session.get(attach) + response = session.get(attach) + return self._check_response(response) def get_yara(self, event_id): to_post = {'request': {'eventid': event_id, 'type': 'yara'}} @@ -957,7 +947,8 @@ class PyMISP(object): """ suricata_rules = urljoin(self.root_url, 'events/nids/suricata/download') session = self.__prepare_session('rules') - return session.get(suricata_rules) + response = session.get(suricata_rules) + return self._check_response(response) def download_suricata_rule_event(self, event_id): """ @@ -967,7 +958,8 @@ class PyMISP(object): """ template = urljoin(self.root_url, 'events/nids/suricata/download/{}'.format(event_id)) session = self.__prepare_session('rules') - return session.get(template) + response = session.get(template) + return self._check_response(response) # ########## Tags ########## @@ -1036,28 +1028,29 @@ class PyMISP(object): session = self.__prepare_session('txt') url = urljoin(self.root_url, 'attributes/text/download/%s' % type_attr) response = session.get(url) - return response + return self._check_response(response) # ############## Statistics ################## - def get_attributes_statistics(self, context='type', percentage=None, force_out=None): + def get_attributes_statistics(self, context='type', percentage=None): """ Get attributes statistics from the MISP instance """ - session = self.__prepare_session(force_out) + session = self.__prepare_session() if (context != 'category'): context = 'type' if percentage is not None: url = urljoin(self.root_url, 'attributes/attributeStatistics/{}/{}'.format(context, percentage)) else: url = urljoin(self.root_url, 'attributes/attributeStatistics/{}'.format(context)) - return session.get(url).json() + response = session.get(url) + return self._check_response(response) - def get_tags_statistics(self, percentage=None, name_sort=None, force_out=None): + def get_tags_statistics(self, percentage=None, name_sort=None): """ Get tags statistics from the MISP instance """ - session = self.__prepare_session(force_out) + session = self.__prepare_session() if percentage is not None: percentage = 'true' else: @@ -1067,55 +1060,34 @@ class PyMISP(object): else: name_sort = 'false' url = urljoin(self.root_url, 'tags/tagStatistics/{}/{}'.format(percentage, name_sort)) - return session.get(url).json() + response = session.get(url).json() + return self._check_response(response) # ############## Sightings ################## - def sighting_per_id(self, attribute_id, force_out=None): - session = self.__prepare_session(force_out) + def sighting_per_id(self, attribute_id): + session = self.__prepare_session() url = urljoin(self.root_url, 'sightings/add/{}'.format(attribute_id)) - return session.post(url) + response = session.post(url) + return self._check_response(response) - def sighting_per_uuid(self, attribute_uuid, force_out=None): - session = self.__prepare_session(force_out) + def sighting_per_uuid(self, attribute_uuid): + session = self.__prepare_session() url = urljoin(self.root_url, 'sightings/add/{}'.format(attribute_uuid)) - return session.post(url) + response = session.post(url) + return self._check_response(response) - def sighting_per_json(self, json_file, force_out=None): - session = self.__prepare_session(force_out) + def sighting_per_json(self, json_file): + session = self.__prepare_session() jdata = json.load(open(json_file)) url = urljoin(self.root_url, 'sightings/add/') - return session.post(url, data=json.dumps(jdata)) + response = session.post(url, data=json.dumps(jdata)) + return self._check_response(response) # ############## Sharing Groups ################## def get_sharing_groups(self): - session = self.__prepare_session(force_out=None) + session = self.__prepare_session() url = urljoin(self.root_url, 'sharing_groups/index.json') response = session.get(url) return self._check_response(response)['response'][0] - - # ############## Deprecated (Pure XML API should not be used) ################## - @deprecated - def download_all(self): - """ - Download all event from the instance - """ - xml = urljoin(self.root_url, 'events/xml/download') - session = self.__prepare_session('xml') - return session.get(xml) - - @deprecated - def download(self, event_id, with_attachement=False): - """ - Download one event in XML - - :param event_id: Event id of the event to download (same as get) - """ - if with_attachement: - attach = 'true' - else: - attach = 'false' - template = urljoin(self.root_url, 'events/xml/download/{}/{}'.format(event_id, attach)) - session = self.__prepare_session('xml') - return session.get(template) From 45379a1158ad922b160914c4446ae92d931c5b3c Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Wed, 10 Aug 2016 15:15:55 +0200 Subject: [PATCH 085/223] Fix prints in tests Signed-off-by: Sebastian Wagner --- tests/test.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test.py b/tests/test.py index 5d7c5f1..e05b857 100755 --- a/tests/test.py +++ b/tests/test.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +from __future__ import print_function from pymisp import PyMISP from keys import url, key @@ -47,7 +48,7 @@ class TestBasic(unittest.TestCase): u'Org': {u'name': u'ORGNAME'}, u'Orgc': {u'name': u'ORGNAME'}, u'threat_level_id': u'1'}} - print event + print(event) self.assertEqual(event, to_check, 'Failed at creating a new Event') return int(event_id) @@ -99,15 +100,15 @@ class TestBasic(unittest.TestCase): def delete(self, eventid): event = self.misp.delete_event(eventid) - print event.json() + print(event.json()) def delete_attr(self, attrid): event = self.misp.delete_attribute(attrid) - print event.json() + print(event.json()) def get(self, eventid): event = self.misp.get_event(eventid) - print event.json() + print(event.json()) def get_stix(self, **kwargs): event = self.misp.get_stix(kwargs) @@ -129,7 +130,7 @@ class TestBasic(unittest.TestCase): u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}], u'proposal_email_lock': False, u'threat_level_id': u'1'}} event = self.misp.add_event(event) - print event.json() + print(event.json()) def test_create_event(self): eventid = self.new_event() From ab09c0a1dc39e12b7abc5c6e8c88cde52c8d1ac0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 11 Aug 2016 17:45:32 +0200 Subject: [PATCH 086/223] Fix calls to __prepare_session Fix #58 --- examples/suricata.py | 2 +- pymisp/api.py | 40 ++++++++++++++++++++-------------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/examples/suricata.py b/examples/suricata.py index 7fd8d78..b1616e8 100755 --- a/examples/suricata.py +++ b/examples/suricata.py @@ -7,7 +7,7 @@ import argparse def init(url, key): - return PyMISP(url, key, True, 'json') + return PyMISP(url, key, True) def fetch(m, all_events, event): diff --git a/pymisp/api.py b/pymisp/api.py index a9979fe..01bdf62 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -126,7 +126,7 @@ class PyMISP(object): self.types = self.describe_types['result']['types'] self.category_type_mapping = self.describe_types['result']['category_type_mappings'] - def __prepare_session(self): + def __prepare_session(self, output='json'): """ Prepare the headers of the session """ @@ -137,8 +137,8 @@ class PyMISP(object): session.proxies = self.proxies session.headers.update( {'Authorization': self.key, - 'Accept': 'application/json', - 'content-type': 'application/json'}) + 'Accept': 'application/{}'.format(output), + 'content-type': 'application/{}'.format(output)}) return session def flatten_error_messages(self, response): @@ -378,13 +378,13 @@ class PyMISP(object): return self._check_response(response) def add_tag(self, event, tag): - session = self.__prepare_session('json') + session = self.__prepare_session() to_post = {'request': {'Event': {'id': event['Event']['id'], 'tag': tag}}} response = session.post(urljoin(self.root_url, 'events/addTag'), data=json.dumps(to_post)) return self._check_response(response) def remove_tag(self, event, tag): - session = self.__prepare_session('json') + session = self.__prepare_session() to_post = {'request': {'Event': {'id': event['Event']['id'], 'tag': tag}}} response = session.post(urljoin(self.root_url, 'events/removeTag'), data=json.dumps(to_post)) return self._check_response(response) @@ -694,7 +694,7 @@ class PyMISP(object): return self._upload_sample(to_post) def _upload_sample(self, to_post): - session = self.__prepare_session('json') + session = self.__prepare_session() url = urljoin(self.root_url, 'events/upload_sample') response = session.post(url, data=json.dumps(to_post)) return self._check_response(response) @@ -719,7 +719,7 @@ class PyMISP(object): return self._check_response(response) def proposal_view(self, event_id=None, proposal_id=None): - session = self.__prepare_session('json') + session = self.__prepare_session() if proposal_id is not None and event_id is not None: return {'error': 'You can only view an event ID or a proposal ID'} if event_id is not None: @@ -729,19 +729,19 @@ class PyMISP(object): return self.__query_proposal(session, 'view', id) def proposal_add(self, event_id, attribute): - session = self.__prepare_session('json') + session = self.__prepare_session() return self.__query_proposal(session, 'add', event_id, attribute) def proposal_edit(self, attribute_id, attribute): - session = self.__prepare_session('json') + session = self.__prepare_session() return self.__query_proposal(session, 'edit', attribute_id, attribute) def proposal_accept(self, proposal_id): - session = self.__prepare_session('json') + session = self.__prepare_session() return self.__query_proposal(session, 'accept', proposal_id) def proposal_discard(self, proposal_id): - session = self.__prepare_session('json') + session = self.__prepare_session() return self.__query_proposal(session, 'discard', proposal_id) # ############################## @@ -798,14 +798,14 @@ class PyMISP(object): buildup_url += '/search{}:{}'.format(rule, joined) else: buildup_url += '/search{}:{}'.format(rule, allowed[rule]) - session = self.__prepare_session('json') + session = self.__prepare_session() url = urljoin(self.root_url, buildup_url) response = session.get(url) return self._check_response(response) def search_all(self, value): query = {'value': value, 'searchall': 1} - session = self.__prepare_session('json') + session = self.__prepare_session() return self.__query(session, 'restSearch/download', query) def __prepare_rest_search(self, values, not_values): @@ -876,7 +876,7 @@ class PyMISP(object): if last is not None: query['last'] = last - session = self.__prepare_session('json') + session = self.__prepare_session() return self.__query(session, 'restSearch/download', query, controller) def get_attachement(self, event_id): @@ -887,13 +887,13 @@ class PyMISP(object): be fetched """ attach = urljoin(self.root_url, 'attributes/downloadAttachment/download/{}'.format(event_id)) - session = self.__prepare_session('json') + session = self.__prepare_session() response = session.get(attach) return self._check_response(response) def get_yara(self, event_id): to_post = {'request': {'eventid': event_id, 'type': 'yara'}} - session = self.__prepare_session('json') + session = self.__prepare_session() response = session.post(urljoin(self.root_url, 'attributes/restSearch'), data=json.dumps(to_post)) result = self._check_response(response) if result.get('error') is not None: @@ -905,7 +905,7 @@ class PyMISP(object): def download_samples(self, sample_hash=None, event_id=None, all_samples=False): to_post = {'request': {'hash': sample_hash, 'eventID': event_id, 'allSamples': all_samples}} - session = self.__prepare_session('json') + session = self.__prepare_session() response = session.post(urljoin(self.root_url, 'attributes/downloadSample'), data=json.dumps(to_post)) result = self._check_response(response) if result.get('error') is not None: @@ -964,7 +964,7 @@ class PyMISP(object): # ########## Tags ########## def get_all_tags(self, quiet=False): - session = self.__prepare_session('json') + session = self.__prepare_session() url = urljoin(self.root_url, 'tags') response = session.get(url) r = self._check_response(response) @@ -978,7 +978,7 @@ class PyMISP(object): def new_tag(self, name=None, colour="#00ace6", exportable=False): to_post = {'Tag': {'name': name, 'colour': colour, 'exportable': exportable}} - session = self.__prepare_session('json') + session = self.__prepare_session() url = urljoin(self.root_url, 'tags/add') response = session.post(url, data=json.dumps(to_post)) return self._check_response(response) @@ -1006,7 +1006,7 @@ class PyMISP(object): """ Returns the version of the instance. """ - session = self.__prepare_session('json') + session = self.__prepare_session() url = urljoin(self.root_url, 'servers/getVersion') response = session.get(url) return self._check_response(response) From f19520d5acde1783004b3edf03db935c26e2ec9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 11 Aug 2016 17:50:47 +0200 Subject: [PATCH 087/223] Proper support of functions returning plain text instead of json --- pymisp/api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 01bdf62..d226ff0 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -948,7 +948,7 @@ class PyMISP(object): suricata_rules = urljoin(self.root_url, 'events/nids/suricata/download') session = self.__prepare_session('rules') response = session.get(suricata_rules) - return self._check_response(response) + return response def download_suricata_rule_event(self, event_id): """ @@ -959,7 +959,7 @@ class PyMISP(object): template = urljoin(self.root_url, 'events/nids/suricata/download/{}'.format(event_id)) session = self.__prepare_session('rules') response = session.get(template) - return self._check_response(response) + return response # ########## Tags ########## @@ -1028,7 +1028,7 @@ class PyMISP(object): session = self.__prepare_session('txt') url = urljoin(self.root_url, 'attributes/text/download/%s' % type_attr) response = session.get(url) - return self._check_response(response) + return response # ############## Statistics ################## From 8ada05f310f69e0e8446e3e322ecac3d86833fa5 Mon Sep 17 00:00:00 2001 From: Thomas King Date: Thu, 11 Aug 2016 19:30:31 +0200 Subject: [PATCH 088/223] Fixed double status code check on helpers and other functions --- pymisp/api.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index d226ff0..5dd0f0e 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -352,30 +352,30 @@ class PyMISP(object): # ########## Helpers ########## def get(self, eid): - response = self.get_event(int(eid), 'json') - return self._check_response(response) + response = self.get_event(int(eid)) + return response def get_stix(self, **kwargs): response = self.get_stix_event(**kwargs) - return self._check_response(response) + return response def update(self, event): eid = event['Event']['id'] - response = self.update_event(eid, event, 'json') - return self._check_response(response) + response = self.update_event(eid, event) + return response def new_event(self, distribution=None, threat_level_id=None, analysis=None, info=None, date=None, published=False): data = self._prepare_full_event(distribution, threat_level_id, analysis, info, date, published) - response = self.add_event(data, 'json') - return self._check_response(response) + response = self.add_event(data) + return response def publish(self, event): if event['Event']['published']: return {'error': 'Already published'} event = self._prepare_update(event) event['Event']['published'] = True - response = self.update_event(event['Event']['id'], event, 'json') - return self._check_response(response) + response = self.update_event(event['Event']['id'], event) + return response def add_tag(self, event, tag): session = self.__prepare_session() @@ -393,7 +393,7 @@ class PyMISP(object): event['Event']['threat_level_id'] = threat_level_id self._prepare_update(event) response = self.update_event(event['Event']['id'], event) - return self._check_response(response) + return response # ##### File attributes ##### @@ -406,8 +406,8 @@ class PyMISP(object): if a.get('distribution') is None: a['distribution'] = 5 event['Event']['Attribute'] = attributes - response = self.update_event(event['Event']['id'], event, 'json') - return self._check_response(response) + response = self.update_event(event['Event']['id'], event) + return response def add_named_attribute(self, event, category, type_value, value, to_ids=False, comment=None, distribution=None, proposal=False): attributes = [] From 17417bd82616767e9589a03e2bdd34e77d5eed8e Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Fri, 12 Aug 2016 13:48:45 +0200 Subject: [PATCH 089/223] Add et2misp example --- examples/et2misp.py | 122 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 examples/et2misp.py diff --git a/examples/et2misp.py b/examples/et2misp.py new file mode 100644 index 0000000..dc6b79e --- /dev/null +++ b/examples/et2misp.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copy Emerging Threats Block IPs list to several MISP events +# Because of the large size of the list the first run will take about 30 mins +# Running it again will update the MISP events if changes are detected +# +# This script requires PyMISP 2.4.50 or later + +import sys, json, time, requests +from pymisp import PyMISP +from keys import misp_url, misp_key + +et_url = 'https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt' +et_str = 'Emerging Threats ' + +def init_misp(): + global mymisp + mymisp = PyMISP(misp_url, misp_key) + +def load_misp_event(eid): + global et_attr + global et_drev + global et_event + et_attr = {} + et_drev = {} + + et_event = mymisp.get(eid) + echeck(et_event) + for a in et_event['Event']['Attribute']: + if a['category'] == 'Network activity': + et_attr[a['value']] = a['id'] + continue + if a['category'] == 'Internal reference': + et_drev = a; + +def init_et(): + global et_data + global et_rev + requests.packages.urllib3.disable_warnings() + s = requests.Session() + r = s.get(et_url) + if r.status_code != 200: + raise Exception('Error getting ET data: {}'.format(r.text)) + name = '' + et_data = {} + et_rev = 0 + for line in r.text.splitlines(): + if line.startswith('# Rev '): + et_rev = int(line[6:]) + continue + if line.startswith('#'): + name = line[1:].strip() + if et_rev and not et_data.get(name): + et_data[name] = {} + continue + l = line.rstrip() + if l: + et_data[name][l] = name + +def update_et_event(name): + if et_drev and et_rev and int(et_drev['value']) < et_rev: + # Copy MISP attributes to new dict + et_ips = dict.fromkeys(et_attr.keys()) + + # Weed out attributes still in ET data + for k,v in et_data[name].items(): + et_attr.pop(k, None) + + # Delete the leftover attributes from MISP + for k,v in et_attr.items(): + r = mymisp.delete_attribute(v) + if r.get('errors'): + print "Error deleting attribute {} ({}): {}\n".format(v,k,r['errors']) + + # Weed out ips already in the MISP event + for k,v in et_ips.items(): + et_data[name].pop(k, None) + + # Add new attributes to MISP event + for k,v in et_data[name].items(): + r = mymisp.add_ipdst(et_event, k, comment=v) + echeck(r, et_event['Event']['id']) + + # Update revision number + et_drev['value'] = et_rev + et_drev.pop('timestamp', None) + attr = [] + attr.append(et_drev) + + # Publish updated MISP event + et_event['Event']['Attribute'] = attr + et_event['Event']['published'] = False + et_event['Event']['date'] = time.strftime('%Y-%m-%d') + r = mymisp.publish(et_event) + echeck(r, et_event['Event']['id']) + +def echeck(r, eid=None): + if r.get('errors'): + if eid: + print "Processing event {} failed: {}".format(eid, r['errors']) + else: + print r['errors'] + sys.exit(1) + +if __name__ == '__main__': + init_misp() + init_et() + + for et_type in set(et_data.keys()): + info = et_str + et_type + r = mymisp.search_index(eventinfo=info) + if r['response']: + eid=r['response'][0]['id'] + else: # event not found, create it + new_event = mymisp.new_event(info=info, distribution=3, threat_level_id=4, analysis=1) + echeck(new_event) + eid=new_event['Event']['id'] + r = mymisp.add_internal_text(new_event, 1, comment='Emerging Threats revision number') + echeck(r, eid) + load_misp_event(eid) + update_et_event(et_type) From 27225403221c2d6608b91243a45953c8b9b6066d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 12 Aug 2016 14:30:50 +0200 Subject: [PATCH 090/223] Fix tests --- tests/test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test.py b/tests/test.py index e05b857..a2afbec 100755 --- a/tests/test.py +++ b/tests/test.py @@ -100,15 +100,15 @@ class TestBasic(unittest.TestCase): def delete(self, eventid): event = self.misp.delete_event(eventid) - print(event.json()) + print(event) def delete_attr(self, attrid): event = self.misp.delete_attribute(attrid) - print(event.json()) + print(event) def get(self, eventid): event = self.misp.get_event(eventid) - print(event.json()) + print(event) def get_stix(self, **kwargs): event = self.misp.get_stix(kwargs) @@ -130,7 +130,7 @@ class TestBasic(unittest.TestCase): u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}], u'proposal_email_lock': False, u'threat_level_id': u'1'}} event = self.misp.add_event(event) - print(event.json()) + print(event) def test_create_event(self): eventid = self.new_event() From f23a7c33579e2e3d1508ecf18c5276dd352bb69b Mon Sep 17 00:00:00 2001 From: Jurriaan Bremer Date: Tue, 16 Aug 2016 11:44:08 +0200 Subject: [PATCH 091/223] allow multiple attributes to be sent off at once Slightly worked out version of the suggestion by doomedraven in #42. --- pymisp/api.py | 123 +++++++++++++++++++++++++++++++++++--------------- tests/test.py | 3 ++ 2 files changed, 90 insertions(+), 36 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 5dd0f0e..6101cda 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -349,6 +349,10 @@ class PyMISP(object): event['Event']['id'] = int(event['Event']['id']) return event + def _one_or_more(self, value): + """Returns a list/tuple of one or more items, regardless of input.""" + return value if isinstance(value, (tuple, list)) else (value,) + # ########## Helpers ########## def get(self, eid): @@ -443,22 +447,23 @@ class PyMISP(object): def av_detection_link(self, event, link, category='Antivirus detection', to_ids=False, comment=None, distribution=None, proposal=False): attributes = [] - attributes.append(self._prepare_full_attribute(category, 'link', link, to_ids, comment, distribution)) + for link in self._one_or_more(link): + attributes.append(self._prepare_full_attribute(category, 'link', link, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) def add_detection_name(self, event, name, category='Antivirus detection', to_ids=False, comment=None, distribution=None, proposal=False): attributes = [] - attributes.append(self._prepare_full_attribute(category, 'text', name, to_ids, comment, distribution)) + for name in self._one_or_more(name): + attributes.append(self._prepare_full_attribute(category, 'text', name, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) def add_filename(self, event, filename, category='Artifacts dropped', to_ids=False, comment=None, distribution=None, proposal=False): attributes = [] - attributes.append(self._prepare_full_attribute(category, 'filename', filename, to_ids, comment, distribution)) + for filename in self._one_or_more(filename): + attributes.append(self._prepare_full_attribute(category, 'filename', filename, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) def add_regkey(self, event, regkey, rvalue=None, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False): - type_value = '{}' - value = '{}' if rvalue: type_value = 'regkey|value' value = '{}|{}'.format(regkey, rvalue) @@ -470,20 +475,36 @@ class PyMISP(object): attributes.append(self._prepare_full_attribute(category, type_value, value, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) + def add_regkeys(self, event, regkeys_values, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False): + attributes = [] + + for regkey, rvalue in regkeys_values.items(): + if rvalue: + type_value = 'regkey|value' + value = '{}|{}'.format(regkey, rvalue) + else: + type_value = 'regkey' + value = regkey + + attributes.append(self._prepare_full_attribute(category, type_value, value, to_ids, comment, distribution)) + return self._send_attributes(event, attributes, proposal) + def add_pattern(self, event, pattern, in_file=True, in_memory=False, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False): attributes = [] - if in_file: - attributes.append(self._prepare_full_attribute(category, 'pattern-in-file', pattern, to_ids, comment, distribution)) - if in_memory: - attributes.append(self._prepare_full_attribute(category, 'pattern-in-memory', pattern, to_ids, comment, distribution)) + for pattern in self._one_or_more(pattern): + if in_file: + attributes.append(self._prepare_full_attribute(category, 'pattern-in-file', pattern, to_ids, comment, distribution)) + if in_memory: + attributes.append(self._prepare_full_attribute(category, 'pattern-in-memory', pattern, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) def add_pipe(self, event, named_pipe, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False): attributes = [] - if not named_pipe.startswith('\\.\\pipe\\'): - named_pipe = '\\.\\pipe\\{}'.format(named_pipe) - attributes.append(self._prepare_full_attribute(category, 'named pipe', named_pipe, to_ids, comment, distribution)) + for named_pipe in self._one_or_more(named_pipe): + if not named_pipe.startswith('\\.\\pipe\\'): + named_pipe = '\\.\\pipe\\{}'.format(named_pipe) + attributes.append(self._prepare_full_attribute(category, 'named pipe', named_pipe, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) def add_mutex(self, event, mutex, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False): @@ -495,29 +516,34 @@ class PyMISP(object): def add_yara(self, event, yara, category='Payload delivery', to_ids=False, comment=None, distribution=None, proposal=False): attributes = [] - attributes.append(self._prepare_full_attribute(category, 'yara', yara, to_ids, comment, distribution)) + for yara in self._one_or_more(yara): + attributes.append(self._prepare_full_attribute(category, 'yara', yara, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) # ##### Network attributes ##### def add_ipdst(self, event, ipdst, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): attributes = [] - attributes.append(self._prepare_full_attribute(category, 'ip-dst', ipdst, to_ids, comment, distribution)) + for ipdst in self._one_or_more(ipdst): + attributes.append(self._prepare_full_attribute(category, 'ip-dst', ipdst, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) def add_ipsrc(self, event, ipsrc, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): attributes = [] - attributes.append(self._prepare_full_attribute(category, 'ip-src', ipsrc, to_ids, comment, distribution)) + for ipsrc in self._one_or_more(ipsrc): + attributes.append(self._prepare_full_attribute(category, 'ip-src', ipsrc, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) def add_hostname(self, event, hostname, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): attributes = [] - attributes.append(self._prepare_full_attribute(category, 'hostname', hostname, to_ids, comment, distribution)) + for hostname in self._one_or_more(hostname): + attributes.append(self._prepare_full_attribute(category, 'hostname', hostname, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) def add_domain(self, event, domain, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): attributes = [] - attributes.append(self._prepare_full_attribute(category, 'domain', domain, to_ids, comment, distribution)) + for domain in self._one_or_more(domain): + attributes.append(self._prepare_full_attribute(category, 'domain', domain, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) def add_domain_ip(self, event, domain, ip, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): @@ -525,107 +551,132 @@ class PyMISP(object): attributes.append(self._prepare_full_attribute(category, 'domain|ip', "%s|%s" % (domain, ip), to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) + def add_domains_ips(self, event, domain_ips, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): + attributes = [] + for domain, ip in domain_ips.items(): + attributes.append(self._prepare_full_attribute(category, 'domain|ip', "%s|%s" % (domain, ip), to_ids, comment, distribution)) + return self._send_attributes(event, attributes, proposal) + def add_url(self, event, url, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): attributes = [] - attributes.append(self._prepare_full_attribute(category, 'url', url, to_ids, comment, distribution)) + for url in self._one_or_more(url): + attributes.append(self._prepare_full_attribute(category, 'url', url, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) def add_useragent(self, event, useragent, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): attributes = [] - attributes.append(self._prepare_full_attribute(category, 'user-agent', useragent, to_ids, comment, distribution)) + for useragent in self._one_or_more(useragent): + attributes.append(self._prepare_full_attribute(category, 'user-agent', useragent, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) def add_traffic_pattern(self, event, pattern, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): attributes = [] - attributes.append(self._prepare_full_attribute(category, 'pattern-in-traffic', pattern, to_ids, comment, distribution)) + for pattern in self._one_or_more(pattern): + attributes.append(self._prepare_full_attribute(category, 'pattern-in-traffic', pattern, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) def add_snort(self, event, snort, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): attributes = [] - attributes.append(self._prepare_full_attribute(category, 'snort', snort, to_ids, comment, distribution)) + for snort in self._one_or_more(snort): + attributes.append(self._prepare_full_attribute(category, 'snort', snort, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) # ##### Email attributes ##### def add_email_src(self, event, email, to_ids=True, comment=None, distribution=None, proposal=False): attributes = [] - attributes.append(self._prepare_full_attribute('Payload delivery', 'email-src', email, to_ids, comment, distribution)) + for email in self._one_or_more(email): + attributes.append(self._prepare_full_attribute('Payload delivery', 'email-src', email, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) def add_email_dst(self, event, email, category='Payload delivery', to_ids=True, comment=None, distribution=None, proposal=False): attributes = [] - attributes.append(self._prepare_full_attribute(category, 'email-dst', email, to_ids, comment, distribution)) + for email in self._one_or_more(email): + attributes.append(self._prepare_full_attribute(category, 'email-dst', email, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) def add_email_subject(self, event, email, to_ids=True, comment=None, distribution=None, proposal=False): attributes = [] - attributes.append(self._prepare_full_attribute('Payload delivery', 'email-subject', email, to_ids, comment, distribution)) + for email in self._one_or_more(email): + attributes.append(self._prepare_full_attribute('Payload delivery', 'email-subject', email, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) def add_email_attachment(self, event, email, to_ids=True, comment=None, distribution=None, proposal=False): attributes = [] - attributes.append(self._prepare_full_attribute('Payload delivery', 'email-attachment', email, to_ids, comment, distribution)) + for email in self._one_or_more(email): + attributes.append(self._prepare_full_attribute('Payload delivery', 'email-attachment', email, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) # ##### Target attributes ##### def add_target_email(self, event, target, to_ids=True, comment=None, distribution=None, proposal=False): attributes = [] - attributes.append(self._prepare_full_attribute('Targeting data', 'target-email', target, to_ids, comment, distribution)) + for target in self._one_or_more(target): + attributes.append(self._prepare_full_attribute('Targeting data', 'target-email', target, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) def add_target_user(self, event, target, to_ids=True, comment=None, distribution=None, proposal=False): attributes = [] - attributes.append(self._prepare_full_attribute('Targeting data', 'target-user', target, to_ids, comment, distribution)) + for target in self._one_or_more(target): + attributes.append(self._prepare_full_attribute('Targeting data', 'target-user', target, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) def add_target_machine(self, event, target, to_ids=True, comment=None, distribution=None, proposal=False): attributes = [] - attributes.append(self._prepare_full_attribute('Targeting data', 'target-machine', target, to_ids, comment, distribution)) + for target in self._one_or_more(target): + attributes.append(self._prepare_full_attribute('Targeting data', 'target-machine', target, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) def add_target_org(self, event, target, to_ids=True, comment=None, distribution=None, proposal=False): attributes = [] - attributes.append(self._prepare_full_attribute('Targeting data', 'target-org', target, to_ids, comment, distribution)) + for target in self._one_or_more(target): + attributes.append(self._prepare_full_attribute('Targeting data', 'target-org', target, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) def add_target_location(self, event, target, to_ids=True, comment=None, distribution=None, proposal=False): attributes = [] - attributes.append(self._prepare_full_attribute('Targeting data', 'target-location', target, to_ids, comment, distribution)) + for target in self._one_or_more(target): + attributes.append(self._prepare_full_attribute('Targeting data', 'target-location', target, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) def add_target_external(self, event, target, to_ids=True, comment=None, distribution=None, proposal=False): attributes = [] - attributes.append(self._prepare_full_attribute('Targeting data', 'target-external', target, to_ids, comment, distribution)) + for target in self._one_or_more(target): + attributes.append(self._prepare_full_attribute('Targeting data', 'target-external', target, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) # ##### Attribution attributes ##### def add_threat_actor(self, event, target, to_ids=True, comment=None, distribution=None, proposal=False): attributes = [] - attributes.append(self._prepare_full_attribute('Attribution', 'threat-actor', target, to_ids, comment, distribution)) + for target in self._one_or_more(target): + attributes.append(self._prepare_full_attribute('Attribution', 'threat-actor', target, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) # ##### Internal reference attributes ##### def add_internal_link(self, event, reference, to_ids=False, comment=None, distribution=None, proposal=False): attributes = [] - attributes.append(self._prepare_full_attribute('Internal reference', 'link', reference, to_ids, comment, distribution)) + for reference in self._one_or_more(reference): + attributes.append(self._prepare_full_attribute('Internal reference', 'link', reference, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) def add_internal_comment(self, event, reference, to_ids=False, comment=None, distribution=None, proposal=False): attributes = [] - attributes.append(self._prepare_full_attribute('Internal reference', 'comment', reference, to_ids, comment, distribution)) + for reference in self._one_or_more(reference): + attributes.append(self._prepare_full_attribute('Internal reference', 'comment', reference, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) def add_internal_text(self, event, reference, to_ids=False, comment=None, distribution=None, proposal=False): attributes = [] - attributes.append(self._prepare_full_attribute('Internal reference', 'text', reference, to_ids, comment, distribution)) + for reference in self._one_or_more(reference): + attributes.append(self._prepare_full_attribute('Internal reference', 'text', reference, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) def add_internal_other(self, event, reference, to_ids=False, comment=None, distribution=None, proposal=False): attributes = [] - attributes.append(self._prepare_full_attribute('Internal reference', 'other', reference, to_ids, comment, distribution)) + for reference in self._one_or_more(reference): + attributes.append(self._prepare_full_attribute('Internal reference', 'other', reference, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) # ################################################## diff --git a/tests/test.py b/tests/test.py index a2afbec..cef2b5f 100755 --- a/tests/test.py +++ b/tests/test.py @@ -156,6 +156,9 @@ class TestBasic(unittest.TestCase): time.sleep(1) self.delete(eventid) + def test_one_or_more(self): + self.assertEqual(self.misp._one_or_more(1), (1,)) + self.assertEqual(self.misp._one_or_more([1]), [1]) if __name__ == '__main__': unittest.main() From 353d04eab3ea41a4cb54c7595c8e4cbf29394791 Mon Sep 17 00:00:00 2001 From: Jurriaan Bremer Date: Tue, 16 Aug 2016 16:51:35 +0200 Subject: [PATCH 092/223] magic value enumerations --- pymisp/api.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/pymisp/api.py b/pymisp/api.py index 6101cda..79ffc1b 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -33,6 +33,29 @@ except NameError: basestring = str +class distributions(object): + """Enumeration of the available distributions.""" + your_organization = 0 + this_community = 1 + connected_communities = 2 + all_communities = 3 + + +class threat_level(object): + """Enumeration of the available threat levels.""" + high = 1 + medium = 2 + low = 3 + undefined = 4 + + +class analysis(object): + """Enumeration of the available analysis statuses.""" + initial = 0 + ongoing = 1 + completed = 2 + + class PyMISPError(Exception): def __init__(self, message): super(PyMISPError, self).__init__(message) @@ -95,6 +118,11 @@ class PyMISP(object): :param proxies: Proxy dict as describes here: http://docs.python-requests.org/en/master/user/advanced/#proxies """ + # So it can may be accessed from the misp object. + distributions = distributions + threat_level = threat_level + analysis = analysis + def __init__(self, url, key, ssl=True, out_type='json', debug=False, proxies=None): if not url: raise NoURL('Please provide the URL of your MISP instance.') From 1ff08c7d737e0f12b555422a847f703ac46a32c3 Mon Sep 17 00:00:00 2001 From: Jurriaan Bremer Date: Tue, 16 Aug 2016 18:35:34 +0200 Subject: [PATCH 093/223] provide sane defaults for upload-sample/samplelist Most of the arguments are unused when a proper event ID has been provided, hence default them to standard values. --- pymisp/api.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 79ffc1b..5ac03ba 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -753,15 +753,17 @@ class PyMISP(object): with open(path, 'rb') as f: return str(base64.b64encode(f.read())) - def upload_sample(self, filename, filepath, event_id, distribution, to_ids, - category, comment, info, analysis, threat_level_id): + def upload_sample(self, filename, filepath, event_id, distribution=None, + to_ids=True, category=None, comment=None, info=None, + analysis=None, threat_level_id=None): to_post = self.prepare_attribute(event_id, distribution, to_ids, category, comment, info, analysis, threat_level_id) to_post['request']['files'] = [{'filename': filename, 'data': self._encode_file_to_upload(filepath)}] return self._upload_sample(to_post) - def upload_samplelist(self, filepaths, event_id, distribution, to_ids, category, - info, analysis, threat_level_id): + def upload_samplelist(self, filepaths, event_id, distribution=None, + to_ids=True, category=None, info=None, + analysis=None, threat_level_id=None): to_post = self.prepare_attribute(event_id, distribution, to_ids, category, info, analysis, threat_level_id) files = [] From 709770e36711f0668d3b4104bf236dbb25359bd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 17 Aug 2016 14:51:35 +0200 Subject: [PATCH 094/223] Version bump --- pymisp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index aea8a6c..be681dc 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,3 +1,3 @@ -__version__ = '2.4.49' +__version__ = '2.4.50' from .api import PyMISP, PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey From 0134e2a9e307593c16e5ec9b5e40a58cfdf54ab3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 17 Aug 2016 18:21:50 +0200 Subject: [PATCH 095/223] Initial version of the offline TestCases Related #56 --- tests/test_offline.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 tests/test_offline.py diff --git a/tests/test_offline.py b/tests/test_offline.py new file mode 100644 index 0000000..fd9fdf6 --- /dev/null +++ b/tests/test_offline.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import requests_mock +import json + +from pymisp import PyMISP + + +@requests_mock.Mocker() +class TestOffline(object): + + def setUp(self): + self.domain = 'http://misp.local/' + self.key = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + self.event = json.load(open('misp_event.json', 'r')) + self.types = json.load(open('describeTypes.json', 'r')) + + def initURI(self, m): + m.register_uri('GET', self.domain + 'servers/getVersion', json={"version": "2.4.50"}) + m.register_uri('GET', self.domain + 'attributes/describeTypes.json', json=self.types) + m.register_uri('GET', self.domain + 'events/2', json=self.event) + + def test_getEvent(self, m): + self.initURI(m) + pymisp = PyMISP(self.domain, self.key, debug=True) + e = pymisp.get_event(2) + print(e) From 24e328f7372314fe4c08af77d7f666c12c303d08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 17 Aug 2016 18:24:17 +0200 Subject: [PATCH 096/223] Add forgotten files --- tests/describeTypes.json | 368 +++++++++++++++++++++++++++++++++++++++ tests/misp_event.json | 78 +++++++++ 2 files changed, 446 insertions(+) create mode 100644 tests/describeTypes.json create mode 100644 tests/misp_event.json diff --git a/tests/describeTypes.json b/tests/describeTypes.json new file mode 100644 index 0000000..bac9389 --- /dev/null +++ b/tests/describeTypes.json @@ -0,0 +1,368 @@ +{ + "result": { + "types": [ + "md5", + "sha1", + "sha256", + "filename", + "pdb", + "filename|md5", + "filename|sha1", + "filename|sha256", + "ip-src", + "ip-dst", + "hostname", + "domain", + "domain|ip", + "email-src", + "email-dst", + "email-subject", + "email-attachment", + "url", + "http-method", + "user-agent", + "regkey", + "regkey|value", + "AS", + "snort", + "pattern-in-file", + "pattern-in-traffic", + "pattern-in-memory", + "yara", + "vulnerability", + "attachment", + "malware-sample", + "link", + "comment", + "text", + "other", + "named pipe", + "mutex", + "target-user", + "target-email", + "target-machine", + "target-org", + "target-location", + "target-external", + "btc", + "iban", + "bic", + "bank-account-nr", + "aba-rtn", + "bin", + "cc-number", + "prtn", + "threat-actor", + "campaign-name", + "campaign-id", + "malware-type", + "uri", + "authentihash", + "ssdeep", + "imphash", + "pehash", + "sha224", + "sha384", + "sha512", + "sha512/224", + "sha512/256", + "tlsh", + "filename|authentihash", + "filename|ssdeep", + "filename|imphash", + "filename|pehash", + "filename|sha224", + "filename|sha384", + "filename|sha512", + "filename|sha512/224", + "filename|sha512/256", + "filename|tlsh", + "windows-scheduled-task", + "windows-service-name", + "windows-service-displayname", + "whois-registrant-email", + "whois-registrant-phone", + "whois-registrant-name", + "whois-registrar", + "whois-creation-date", + "targeted-threat-index", + "mailslot", + "pipe", + "ssl-cert-attributes", + "x509-fingerprint-sha1" + ], + "categories": [ + "Internal reference", + "Targeting data", + "Antivirus detection", + "Payload delivery", + "Artifacts dropped", + "Payload installation", + "Persistence mechanism", + "Network activity", + "Payload type", + "Attribution", + "External analysis", + "Financial fraud", + "Other" + ], + "category_type_mappings": { + "Internal reference": [ + "link", + "comment", + "text", + "other" + ], + "Targeting data": [ + "target-user", + "target-email", + "target-machine", + "target-org", + "target-location", + "target-external", + "comment" + ], + "Antivirus detection": [ + "link", + "comment", + "text", + "attachment", + "other" + ], + "Payload delivery": [ + "md5", + "sha1", + "sha224", + "sha256", + "sha384", + "sha512", + "sha512/224", + "sha512/256", + "ssdeep", + "imphash", + "authentihash", + "pehash", + "tlsh", + "filename", + "filename|md5", + "filename|sha1", + "filename|sha224", + "filename|sha256", + "filename|sha384", + "filename|sha512", + "filename|sha512/224", + "filename|sha512/256", + "filename|authentihash", + "filename|ssdeep", + "filename|tlsh", + "filename|imphash", + "filename|pehash", + "ip-src", + "ip-dst", + "hostname", + "domain", + "email-src", + "email-dst", + "email-subject", + "email-attachment", + "url", + "user-agent", + "AS", + "pattern-in-file", + "pattern-in-traffic", + "yara", + "attachment", + "malware-sample", + "link", + "malware-type", + "comment", + "text", + "vulnerability", + "x509-fingerprint-sha1", + "other" + ], + "Artifacts dropped": [ + "md5", + "sha1", + "sha224", + "sha256", + "sha384", + "sha512", + "sha512/224", + "sha512/256", + "ssdeep", + "imphash", + "authentihash", + "filename", + "filename|md5", + "filename|sha1", + "filename|sha224", + "filename|sha256", + "filename|sha384", + "filename|sha512", + "filename|sha512/224", + "filename|sha512/256", + "filename|authentihash", + "filename|ssdeep", + "filename|tlsh", + "filename|imphash", + "filename|pehash", + "regkey", + "regkey|value", + "pattern-in-file", + "pattern-in-memory", + "pdb", + "yara", + "attachment", + "malware-sample", + "named pipe", + "mutex", + "windows-scheduled-task", + "windows-service-name", + "windows-service-displayname", + "comment", + "text", + "x509-fingerprint-sha1", + "other" + ], + "Payload installation": [ + "md5", + "sha1", + "sha224", + "sha256", + "sha384", + "sha512", + "sha512/224", + "sha512/256", + "ssdeep", + "imphash", + "authentihash", + "pehash", + "tlsh", + "filename", + "filename|md5", + "filename|sha1", + "filename|sha224", + "filename|sha256", + "filename|sha384", + "filename|sha512", + "filename|sha512/224", + "filename|sha512/256", + "filename|authentihash", + "filename|ssdeep", + "filename|tlsh", + "filename|imphash", + "filename|pehash", + "pattern-in-file", + "pattern-in-traffic", + "pattern-in-memory", + "yara", + "vulnerability", + "attachment", + "malware-sample", + "malware-type", + "comment", + "text", + "x509-fingerprint-sha1", + "other" + ], + "Persistence mechanism": [ + "filename", + "regkey", + "regkey|value", + "comment", + "text", + "other" + ], + "Network activity": [ + "ip-src", + "ip-dst", + "hostname", + "domain", + "domain|ip", + "email-dst", + "url", + "uri", + "user-agent", + "http-method", + "AS", + "snort", + "pattern-in-file", + "pattern-in-traffic", + "attachment", + "comment", + "text", + "x509-fingerprint-sha1", + "other" + ], + "Payload type": [ + "comment", + "text", + "other" + ], + "Attribution": [ + "threat-actor", + "campaign-name", + "campaign-id", + "whois-registrant-phone", + "whois-registrant-email", + "whois-registrant-name", + "whois-registrar", + "whois-creation-date", + "comment", + "text", + "x509-fingerprint-sha1", + "other" + ], + "External analysis": [ + "md5", + "sha1", + "sha256", + "filename", + "filename|md5", + "filename|sha1", + "filename|sha256", + "ip-src", + "ip-dst", + "hostname", + "domain", + "domain|ip", + "url", + "user-agent", + "regkey", + "regkey|value", + "AS", + "snort", + "pattern-in-file", + "pattern-in-traffic", + "pattern-in-memory", + "vulnerability", + "attachment", + "malware-sample", + "link", + "comment", + "text", + "x509-fingerprint-sha1", + "other" + ], + "Financial fraud": [ + "btc", + "iban", + "bic", + "bank-account-nr", + "aba-rtn", + "bin", + "cc-number", + "prtn", + "comment", + "text", + "other" + ], + "Other": [ + "comment", + "text", + "other" + ] + } + } +} diff --git a/tests/misp_event.json b/tests/misp_event.json new file mode 100644 index 0000000..1db311a --- /dev/null +++ b/tests/misp_event.json @@ -0,0 +1,78 @@ +{ + "Attribute": [ + { + "ShadowAttribute": [], + "SharingGroup": [], + "category": "Payload delivery", + "comment": "", + "deleted": false, + "distribution": "5", + "event_id": "2", + "id": "7", + "sharing_group_id": "0", + "timestamp": "1465681304", + "to_ids": false, + "type": "url", + "uuid": "575c8598-f1f0-4c16-a94a-0612c0a83866", + "value": "http://fake.website.com/malware/is/here" + }, + { + "ShadowAttribute": [], + "SharingGroup": [], + "category": "Payload type", + "comment": "", + "deleted": false, + "distribution": "5", + "event_id": "2", + "id": "6", + "sharing_group_id": "0", + "timestamp": "1465681801", + "to_ids": false, + "type": "text", + "uuid": "575c8549-9010-4555-8b37-057ac0a83866", + "value": "Locky" + } + ], + "Org": { + "id": "1", + "name": "ORGNAME", + "uuid": "57586e9a-4a64-4f79-9009-4dc1c0a83866" + }, + "Orgc": { + "id": "1", + "name": "ORGNAME", + "uuid": "57586e9a-4a64-4f79-9009-4dc1c0a83866" + }, + "RelatedEvent": [], + "ShadowAttribute": [], + "Tag": [ + { + "colour": "#005a5a", + "exportable": true, + "id": "6", + "name": "ecsirt:malicious-code=\"ransomware\"" + }, + { + "colour": "#142bf7", + "exportable": true, + "id": "1", + "name": "for_intelmq_processing" + } + ], + "analysis": "0", + "attribute_count": "2", + "date": "2016-06-09", + "distribution": "0", + "id": "2", + "info": "A Random Event", + "locked": false, + "org_id": "1", + "orgc_id": "1", + "proposal_email_lock": false, + "publish_timestamp": "0", + "published": false, + "sharing_group_id": "0", + "threat_level_id": "1", + "timestamp": "1465681801", + "uuid": "5758ebf5-c898-48e6-9fe9-5665c0a83866" +} From 33a2f0603e218c04500fd442e169c7253711b0e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 17 Aug 2016 18:36:01 +0200 Subject: [PATCH 097/223] Add travis --- .travis.yml | 24 ++++++++++++++++++++++++ tests/test_offline.py | 4 ++-- 2 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..0ad9dbe --- /dev/null +++ b/.travis.yml @@ -0,0 +1,24 @@ +language: python + +cache: pip + +python: + - "2.7" + - "3.3" + - "3.4" + - "3.5" + - "3.5-dev" + - "nightly" + +install: + - pip install -U nose + - pip install coveralls + - pip install codecov + - pip install . + +script: + - nosetests --with-coverage --cover-package=pymisp tests/test_offline.py + +after_success: + - codecov + - coveralls diff --git a/tests/test_offline.py b/tests/test_offline.py index fd9fdf6..07bcb10 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -13,8 +13,8 @@ class TestOffline(object): def setUp(self): self.domain = 'http://misp.local/' self.key = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' - self.event = json.load(open('misp_event.json', 'r')) - self.types = json.load(open('describeTypes.json', 'r')) + self.event = json.load(open('tests/misp_event.json', 'r')) + self.types = json.load(open('tests/describeTypes.json', 'r')) def initURI(self, m): m.register_uri('GET', self.domain + 'servers/getVersion', json={"version": "2.4.50"}) From 76812f96aa89f650b4bbb6cede39082ef3eb3ad9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 17 Aug 2016 18:38:37 +0200 Subject: [PATCH 098/223] Add dependency --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 0ad9dbe..fa42b95 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ install: - pip install -U nose - pip install coveralls - pip install codecov + - pip install requests-mock - pip install . script: From cdcb1cca5e912703f9f9cb8b9e33869489ccf5e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 18 Aug 2016 00:23:49 +0200 Subject: [PATCH 099/223] Update testing --- README.md | 5 ++++- examples/up.py | 4 ++-- pymisp/api.py | 17 ----------------- tests/test_offline.py | 42 ++++++++++++++++++++++++++++++++++++++---- 4 files changed, 44 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index c60cc2e..bcbfe7a 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,6 @@ +[![Build Status](https://travis-ci.org/MISP/PyMISP.svg?branch=master)](https://travis-ci.org/MISP/PyMISP) +[![Coverage Status](https://coveralls.io/repos/github/MISP/PyMISP/badge.svg?branch=master)](https://coveralls.io/github/MISP/PyMISP?branch=master) + # PyMISP - Python Library to access MISP PyMISP is a Python library to access [MISP](https://github.com/MISP/MISP) platforms via their REST API. @@ -20,7 +23,7 @@ git clone https://github.com/CIRCL/PyMISP.git && cd PyMISP python setup.py install ~~~~ -## Samples and how to use PyMISP +## Samples and how to use PyMISP Various examples and samples scripts are in the [examples/](examples/) directory. diff --git a/examples/up.py b/examples/up.py index f1b5d45..cdca33e 100755 --- a/examples/up.py +++ b/examples/up.py @@ -10,13 +10,13 @@ import argparse def init(url, key): - return PyMISP(url, key, True, 'json') + return PyMISP(url, key, True, 'json', debug=True) def up_event(m, event, content): with open(content, 'r') as f: result = m.update_event(event, f.read()) - print result.text + print(result) if __name__ == '__main__': parser = argparse.ArgumentParser(description='Get an event from a MISP instance.') diff --git a/pymisp/api.py b/pymisp/api.py index 5ac03ba..a59967b 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -86,23 +86,6 @@ class NoKey(PyMISPError): pass -def deprecated(func): - '''This is a decorator which can be used to mark functions - as deprecated. It will result in a warning being emitted - when the function is used.''' - - @functools.wraps(func) - def new_func(*args, **kwargs): - warnings.warn_explicit( - "Call to deprecated function {}.".format(func.__name__), - category=DeprecationWarning, - filename=func.__code__.co_filename, - lineno=func.__code__.co_firstlineno + 1 - ) - return func(*args, **kwargs) - return new_func - - class PyMISP(object): """ Python API for MISP diff --git a/tests/test_offline.py b/tests/test_offline.py index 07bcb10..e9e30ac 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -1,14 +1,16 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +import unittest import requests_mock import json +import pymisp from pymisp import PyMISP @requests_mock.Mocker() -class TestOffline(object): +class TestOffline(unittest.TestCase): def setUp(self): self.domain = 'http://misp.local/' @@ -17,12 +19,44 @@ class TestOffline(object): self.types = json.load(open('tests/describeTypes.json', 'r')) def initURI(self, m): - m.register_uri('GET', self.domain + 'servers/getVersion', json={"version": "2.4.50"}) + m.register_uri('GET', self.domain + 'servers/getVersion', json={"version": pymisp.__version__[1:]}) m.register_uri('GET', self.domain + 'attributes/describeTypes.json', json=self.types) m.register_uri('GET', self.domain + 'events/2', json=self.event) + m.register_uri('POST', self.domain + 'events/2', json=self.event) + m.register_uri('DELETE', self.domain + 'events/2', json={'message': 'Event deleted.'}) + m.register_uri('DELETE', self.domain + 'events/3', json={'errors': ['Invalid event'], 'message': 'Invalid event', 'name': 'Invalid event', 'url': '/events/3'}) + m.register_uri('DELETE', self.domain + 'attribute/2', json={'message': 'Attribute deleted.'}) def test_getEvent(self, m): self.initURI(m) pymisp = PyMISP(self.domain, self.key, debug=True) - e = pymisp.get_event(2) - print(e) + e1 = pymisp.get_event(2) + e2 = pymisp.get(2) + self.assertEqual(e1, e2) + self.assertEqual(self.event, e2) + + def test_updateEvent(self, m): + self.initURI(m) + pymisp = PyMISP(self.domain, self.key, debug=True) + e0 = pymisp.update_event(2, json.dumps(self.event)) + e1 = pymisp.update_event(2, self.event) + self.assertEqual(e0, e1) + e = {'Event': self.event} + e2 = pymisp.update(e) + self.assertEqual(e1, e2) + self.assertEqual(self.event, e2) + + def test_deleteEvent(self, m): + self.initURI(m) + pymisp = PyMISP(self.domain, self.key, debug=True) + d = pymisp.delete_event(2) + self.assertEqual(d, {'message': 'Event deleted.'}) + d = pymisp.delete_event(3) + self.assertEqual(d, {'errors': ['Invalid event'], 'message': 'Invalid event', 'name': 'Invalid event', 'url': '/events/3'}) + + def test_deleteAttribute(self, m): + # FIXME: https://github.com/MISP/MISP/issues/1449 + self.initURI(m) + pymisp = PyMISP(self.domain, self.key, debug=True) + d = pymisp.delete_attribute(2) + self.assertEqual(d, {'message': 'Event deleted.'}) From a4acc5d147e9357528f4b2be53622e50f52d49f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 18 Aug 2016 00:40:30 +0200 Subject: [PATCH 100/223] Add tests --- examples/del.py | 36 ++++++++++++++++++++++++++++++++++++ examples/get.py | 5 ++--- tests/test_offline.py | 20 +++++++++++++++----- 3 files changed, 53 insertions(+), 8 deletions(-) create mode 100755 examples/del.py diff --git a/examples/del.py b/examples/del.py new file mode 100755 index 0000000..0577353 --- /dev/null +++ b/examples/del.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pymisp import PyMISP +from keys import misp_url, misp_key +import argparse + + +# Usage for pipe masters: ./last.py -l 5h | jq . + + +def init(url, key): + return PyMISP(url, key, True, 'json', debug=True) + + +def del_event(m, eventid): + result = m.delete_event(eventid) + print(result) + +def del_attr(m, attrid): + result = m.delete_attribute(attrid) + print(result) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Delete an event from a MISP instance.') + parser.add_argument("-e", "--event", help="Event ID to delete.") + parser.add_argument("-a", "--attribute", help="Attribute ID to delete.") + + args = parser.parse_args() + + misp = init(misp_url, misp_key) + + if args.event: + del_event(misp, args.event) + else: + del_attr(misp, args.attribute) diff --git a/examples/get.py b/examples/get.py index 71bfa97..d2be085 100755 --- a/examples/get.py +++ b/examples/get.py @@ -24,12 +24,11 @@ def init(url, key): def get_event(m, event, out=None): result = m.get_event(event) - r = result.json() if out is None: - print(json.dumps(r) + '\n') + print(json.dumps(result) + '\n') else: with open(out, 'w') as f: - f.write(json.dumps(r) + '\n') + f.write(json.dumps(result) + '\n') if __name__ == '__main__': diff --git a/tests/test_offline.py b/tests/test_offline.py index e9e30ac..26e702c 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -13,9 +13,10 @@ from pymisp import PyMISP class TestOffline(unittest.TestCase): def setUp(self): + self.maxDiff = None self.domain = 'http://misp.local/' self.key = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' - self.event = json.load(open('tests/misp_event.json', 'r')) + self.event = {'Event': json.load(open('tests/misp_event.json', 'r'))} self.types = json.load(open('tests/describeTypes.json', 'r')) def initURI(self, m): @@ -25,7 +26,7 @@ class TestOffline(unittest.TestCase): m.register_uri('POST', self.domain + 'events/2', json=self.event) m.register_uri('DELETE', self.domain + 'events/2', json={'message': 'Event deleted.'}) m.register_uri('DELETE', self.domain + 'events/3', json={'errors': ['Invalid event'], 'message': 'Invalid event', 'name': 'Invalid event', 'url': '/events/3'}) - m.register_uri('DELETE', self.domain + 'attribute/2', json={'message': 'Attribute deleted.'}) + m.register_uri('DELETE', self.domain + 'attributes/2', json={'message': 'Attribute deleted.'}) def test_getEvent(self, m): self.initURI(m) @@ -41,8 +42,7 @@ class TestOffline(unittest.TestCase): e0 = pymisp.update_event(2, json.dumps(self.event)) e1 = pymisp.update_event(2, self.event) self.assertEqual(e0, e1) - e = {'Event': self.event} - e2 = pymisp.update(e) + e2 = pymisp.update(e0) self.assertEqual(e1, e2) self.assertEqual(self.event, e2) @@ -59,4 +59,14 @@ class TestOffline(unittest.TestCase): self.initURI(m) pymisp = PyMISP(self.domain, self.key, debug=True) d = pymisp.delete_attribute(2) - self.assertEqual(d, {'message': 'Event deleted.'}) + self.assertEqual(d, {'message': 'Attribute deleted.'}) + + def test_publish(self, m): + self.initURI(m) + pymisp = PyMISP(self.domain, self.key, debug=True) + e = pymisp.publish(self.event) + pub = self.event + pub['Event']['published'] = True + self.assertEqual(e, pub) + e = pymisp.publish(self.event) + self.assertEqual(e, {'error': 'Already published'}) From 7cbda22667a45c25a33a2ea5d03a7efcd4f46fd9 Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Thu, 18 Aug 2016 11:27:02 +0200 Subject: [PATCH 101/223] Speed up et2misp --- examples/et2misp.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) mode change 100644 => 100755 examples/et2misp.py diff --git a/examples/et2misp.py b/examples/et2misp.py old mode 100644 new mode 100755 index dc6b79e..e45f395 --- a/examples/et2misp.py +++ b/examples/et2misp.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # # Copy Emerging Threats Block IPs list to several MISP events -# Because of the large size of the list the first run will take about 30 mins +# Because of the large size of the list the first run will take a minute # Running it again will update the MISP events if changes are detected # # This script requires PyMISP 2.4.50 or later @@ -78,9 +78,13 @@ def update_et_event(name): et_data[name].pop(k, None) # Add new attributes to MISP event - for k,v in et_data[name].items(): - r = mymisp.add_ipdst(et_event, k, comment=v) - echeck(r, et_event['Event']['id']) + ipdst = [] + for i,k in enumerate(et_data[name].items(), 1-len(et_data[name])): + ipdst.append(k[0]) + if i % 100 == 0: + r = mymisp.add_ipdst(et_event, ipdst) + echeck(r, et_event['Event']['id']) + ipdst = [] # Update revision number et_drev['value'] = et_rev From cdc77de5983a5d28838b7bf33e416429152a8f6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 18 Aug 2016 13:18:58 +0200 Subject: [PATCH 102/223] Add some more tests --- pymisp/api.py | 2 +- tests/sharing_groups.json | 100 ++++++++++++++++++++++++++++++++++++++ tests/test_offline.py | 21 ++++++-- 3 files changed, 119 insertions(+), 4 deletions(-) create mode 100644 tests/sharing_groups.json diff --git a/pymisp/api.py b/pymisp/api.py index a59967b..d40cb04 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1071,7 +1071,7 @@ class PyMISP(object): Returns the version of the instance. """ session = self.__prepare_session() - url = urljoin(self.root_url, 'servers/getVersion') + url = urljoin(self.root_url, 'servers/getVersion.json') response = session.get(url) return self._check_response(response) diff --git a/tests/sharing_groups.json b/tests/sharing_groups.json new file mode 100644 index 0000000..96a3e5f --- /dev/null +++ b/tests/sharing_groups.json @@ -0,0 +1,100 @@ +{ + "response": [ + { + "SharingGroup": { + "id": "1", + "name": "PrivateTrustedGroup", + "description": "", + "releasability": "", + "local": true, + "active": true + }, + "Organisation": { + "id": "1", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "SharingGroupOrg": [ + { + "id": "1", + "sharing_group_id": "1", + "org_id": "1", + "extend": true, + "Organisation": { + "name": "CIRCL", + "id": "1", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + } + }, + { + "id": "2", + "sharing_group_id": "1", + "org_id": "2", + "extend": false, + "Organisation": { + "name": "PifPafPoum", + "id": "2", + "uuid": "56bf12a7-c19c-4b98-83e7-d9bb02de0b81" + } + } + ], + "SharingGroupServer": [ + { + "all_orgs": false, + "server_id": "0", + "sharing_group_id": "1", + "Server": [] + } + ], + "editable": true + }, + { + "SharingGroup": { + "id": "2", + "name": "test", + "description": "", + "releasability": "", + "local": true, + "active": true + }, + "Organisation": { + "id": "1", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "SharingGroupOrg": [ + { + "id": "3", + "sharing_group_id": "2", + "org_id": "1", + "extend": true, + "Organisation": { + "name": "CIRCL", + "id": "1", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + } + }, + { + "id": "4", + "sharing_group_id": "2", + "org_id": "2", + "extend": false, + "Organisation": { + "name": "PifPafPoum", + "id": "2", + "uuid": "56bf12a7-c19c-4b98-83e7-d9bb02de0b81" + } + } + ], + "SharingGroupServer": [ + { + "all_orgs": false, + "server_id": "0", + "sharing_group_id": "2", + "Server": [] + } + ], + "editable": true + } + ] +} diff --git a/tests/test_offline.py b/tests/test_offline.py index 26e702c..afb4f34 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -5,7 +5,7 @@ import unittest import requests_mock import json -import pymisp +import pymisp as pm from pymisp import PyMISP @@ -18,9 +18,11 @@ class TestOffline(unittest.TestCase): self.key = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' self.event = {'Event': json.load(open('tests/misp_event.json', 'r'))} self.types = json.load(open('tests/describeTypes.json', 'r')) + self.sharing_groups = json.load(open('tests/sharing_groups.json', 'r')) def initURI(self, m): - m.register_uri('GET', self.domain + 'servers/getVersion', json={"version": pymisp.__version__[1:]}) + m.register_uri('GET', self.domain + 'servers/getVersion.json', json={"version": "2.4.50"}) + m.register_uri('GET', self.domain + 'sharing_groups/index.json', json=self.sharing_groups) m.register_uri('GET', self.domain + 'attributes/describeTypes.json', json=self.types) m.register_uri('GET', self.domain + 'events/2', json=self.event) m.register_uri('POST', self.domain + 'events/2', json=self.event) @@ -55,7 +57,6 @@ class TestOffline(unittest.TestCase): self.assertEqual(d, {'errors': ['Invalid event'], 'message': 'Invalid event', 'name': 'Invalid event', 'url': '/events/3'}) def test_deleteAttribute(self, m): - # FIXME: https://github.com/MISP/MISP/issues/1449 self.initURI(m) pymisp = PyMISP(self.domain, self.key, debug=True) d = pymisp.delete_attribute(2) @@ -70,3 +71,17 @@ class TestOffline(unittest.TestCase): self.assertEqual(e, pub) e = pymisp.publish(self.event) self.assertEqual(e, {'error': 'Already published'}) + + def test_getVersions(self, m): + self.initURI(m) + pymisp = PyMISP(self.domain, self.key, debug=True) + api_version = pymisp.get_api_version() + self.assertEqual(api_version, {'version': pm.__version__}) + server_version = pymisp.get_version() + self.assertEqual(server_version, {"version": "2.4.50"}) + + def test_getSharingGroups(self, m): + self.initURI(m) + pymisp = PyMISP(self.domain, self.key, debug=True) + sharing_groups = pymisp.get_sharing_groups() + self.assertEqual(sharing_groups, self.sharing_groups['response'][0]) From 0dbf7564ec31676d4d7fc17d15ad2fdbd44e8d47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 19 Aug 2016 10:13:00 +0200 Subject: [PATCH 103/223] add auth error test --- tests/test_offline.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_offline.py b/tests/test_offline.py index afb4f34..1a6497a 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -19,8 +19,12 @@ class TestOffline(unittest.TestCase): self.event = {'Event': json.load(open('tests/misp_event.json', 'r'))} self.types = json.load(open('tests/describeTypes.json', 'r')) self.sharing_groups = json.load(open('tests/sharing_groups.json', 'r')) + self.auth_error_msg = {"name": "Authentication failed. Please make sure you pass the API key of an API enabled user along in the Authorization header.", + "message": "Authentication failed. Please make sure you pass the API key of an API enabled user along in the Authorization header.", + "url": "\/events\/1"} def initURI(self, m): + m.register_uri('GET', self.domain + 'events/1', json=self.auth_error_msg, status_code=403) m.register_uri('GET', self.domain + 'servers/getVersion.json', json={"version": "2.4.50"}) m.register_uri('GET', self.domain + 'sharing_groups/index.json', json=self.sharing_groups) m.register_uri('GET', self.domain + 'attributes/describeTypes.json', json=self.types) @@ -85,3 +89,14 @@ class TestOffline(unittest.TestCase): pymisp = PyMISP(self.domain, self.key, debug=True) sharing_groups = pymisp.get_sharing_groups() self.assertEqual(sharing_groups, self.sharing_groups['response'][0]) + + def test_auth_error(self, m): + self.initURI(m) + pymisp = PyMISP(self.domain, self.key, debug=True) + error = pymisp.get(1) + response = self.auth_error_msg + response['errors'] = [response['message']] + print(error) + +if __name__ == '__main__': + unittest.main() From a1a524c2c55eb035c5bacb65ee38130b90444e06 Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Fri, 26 Aug 2016 09:11:01 +0200 Subject: [PATCH 104/223] Add ssl client certificate support --- pymisp/api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pymisp/api.py b/pymisp/api.py index d40cb04..c89f3ae 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -106,7 +106,7 @@ class PyMISP(object): threat_level = threat_level analysis = analysis - def __init__(self, url, key, ssl=True, out_type='json', debug=False, proxies=None): + def __init__(self, url, key, ssl=True, out_type='json', debug=False, proxies=None, cert=None): if not url: raise NoURL('Please provide the URL of your MISP instance.') if not key: @@ -116,6 +116,7 @@ class PyMISP(object): self.key = key self.ssl = ssl self.proxies = proxies + self.cert = cert if out_type != 'json': raise PyMISPError('The only output type supported by PyMISP is JSON. If you still rely on XML, use PyMISP v2.4.49') self.debug = debug @@ -146,6 +147,7 @@ class PyMISP(object): session = requests.Session() session.verify = self.ssl session.proxies = self.proxies + session.cert = self.cert session.headers.update( {'Authorization': self.key, 'Accept': 'application/{}'.format(output), From 0b0905d0e76b0ac231beb1246046b2253fb1ccbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 26 Aug 2016 09:24:02 +0200 Subject: [PATCH 105/223] Update documentation for client side certificate --- pymisp/api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pymisp/api.py b/pymisp/api.py index c89f3ae..8e173d5 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -99,6 +99,7 @@ class PyMISP(object): :param out_type: Type of object (json) NOTE: XML output isn't supported anymore, keeping the flag for compatibility reasons. :param debug: print all the messages received from the server :param proxies: Proxy dict as describes here: http://docs.python-requests.org/en/master/user/advanced/#proxies + :param proxies: Client certificate, as described there: http://docs.python-requests.org/en/master/user/advanced/#ssl-cert-verification """ # So it can may be accessed from the misp object. From 32bc8782f058f7adacad37caf593330653a9a9ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 26 Aug 2016 12:00:13 +0200 Subject: [PATCH 106/223] Add doc --- docs/Makefile | 225 ++++++++++++++++++++ docs/source/conf.py | 444 ++++++++++++++++++++++++++++++++++++++++ docs/source/index.rst | 22 ++ docs/source/modules.rst | 7 + docs/source/pymisp.rst | 22 ++ pymisp/api.py | 2 +- 6 files changed, 721 insertions(+), 1 deletion(-) create mode 100644 docs/Makefile create mode 100644 docs/source/conf.py create mode 100644 docs/source/index.rst create mode 100644 docs/source/modules.rst create mode 100644 docs/source/pymisp.rst diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..fda38db --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,225 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source + +.PHONY: help +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " applehelp to make an Apple Help Book" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " epub3 to make an epub3" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + @echo " coverage to run coverage check of the documentation (if enabled)" + @echo " dummy to check syntax errors of document sources" + +.PHONY: clean +clean: + rm -rf $(BUILDDIR)/* + +.PHONY: html +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +.PHONY: dirhtml +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +.PHONY: singlehtml +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +.PHONY: pickle +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +.PHONY: json +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +.PHONY: htmlhelp +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +.PHONY: qthelp +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/PyMISP.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/PyMISP.qhc" + +.PHONY: applehelp +applehelp: + $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp + @echo + @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." + @echo "N.B. You won't be able to view it unless you put it in" \ + "~/Library/Documentation/Help or install it in your application" \ + "bundle." + +.PHONY: devhelp +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/PyMISP" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/PyMISP" + @echo "# devhelp" + +.PHONY: epub +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +.PHONY: epub3 +epub3: + $(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3 + @echo + @echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3." + +.PHONY: latex +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +.PHONY: latexpdf +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: latexpdfja +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: text +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +.PHONY: man +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +.PHONY: texinfo +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +.PHONY: info +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +.PHONY: gettext +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +.PHONY: changes +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +.PHONY: linkcheck +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +.PHONY: doctest +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +.PHONY: coverage +coverage: + $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage + @echo "Testing of coverage in the sources finished, look at the " \ + "results in $(BUILDDIR)/coverage/python.txt." + +.PHONY: xml +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +.PHONY: pseudoxml +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." + +.PHONY: dummy +dummy: + $(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy + @echo + @echo "Build finished. Dummy builder generates no files." diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..59888b2 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,444 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# PyMISP documentation build configuration file, created by +# sphinx-quickstart on Fri Aug 26 11:39:17 2016. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + +from recommonmark.parser import CommonMarkParser + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.mathjax', + 'sphinx.ext.ifconfig', + 'sphinx.ext.viewcode', + 'sphinx.ext.githubpages', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +source_parsers = { + '.md': CommonMarkParser, +} + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = ['.rst', '.md'] + +# The encoding of source files. +# +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'PyMISP' +copyright = '2016, Raphaël Vinot' +author = 'Raphaël Vinot' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '2.4.50' +# The full version, including alpha/beta/rc tags. +release = '2.4.50' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# +# today = '' +# +# Else, today_fmt is used as the format for a strftime call. +# +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = [] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +# +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +# keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +# html_theme_path = [] + +# The name for this set of Sphinx documents. +# " v documentation" by default. +# +# html_title = 'PyMISP v2.4.50' + +# A shorter title for the navigation bar. Default is the same as html_title. +# +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# +# html_logo = None + +# The name of an image file (relative to this directory) to use as a favicon of +# the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +# +# html_extra_path = [] + +# If not None, a 'Last updated on:' timestamp is inserted at every page +# bottom, using the given strftime format. +# The empty string is equivalent to '%b %d, %Y'. +# +# html_last_updated_fmt = None + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# +# html_additional_pages = {} + +# If false, no module index is generated. +# +# html_domain_indices = True + +# If false, no index is generated. +# +# html_use_index = True + +# If true, the index is split into individual pages for each letter. +# +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +# +# html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr', 'zh' +# +# html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# 'ja' uses this config value. +# 'zh' user can custom change `jieba` dictionary path. +# +# html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +# +# html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'PyMISPdoc' + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'PyMISP.tex', 'PyMISP Documentation', + 'Raphaël Vinot', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# +# latex_use_parts = False + +# If true, show page references after internal links. +# +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# +# latex_appendices = [] + +# It false, will not define \strong, \code, itleref, \crossref ... but only +# \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added +# packages. +# +# latex_keep_old_macro_names = True + +# If false, no module index is generated. +# +# latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'pymisp', 'PyMISP Documentation', + [author], 1) +] + +# If true, show URL addresses after external links. +# +# man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'PyMISP', 'PyMISP Documentation', + author, 'PyMISP', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +# +# texinfo_appendices = [] + +# If false, no module index is generated. +# +# texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# +# texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +# +# texinfo_no_detailmenu = False + + +# -- Options for Epub output ---------------------------------------------- + +# Bibliographic Dublin Core info. +epub_title = project +epub_author = author +epub_publisher = author +epub_copyright = copyright + +# The basename for the epub file. It defaults to the project name. +# epub_basename = project + +# The HTML theme for the epub output. Since the default themes are not +# optimized for small screen space, using the same theme for HTML and epub +# output is usually not wise. This defaults to 'epub', a theme designed to save +# visual space. +# +# epub_theme = 'epub' + +# The language of the text. It defaults to the language option +# or 'en' if the language is not set. +# +# epub_language = '' + +# The scheme of the identifier. Typical schemes are ISBN or URL. +# epub_scheme = '' + +# The unique identifier of the text. This can be a ISBN number +# or the project homepage. +# +# epub_identifier = '' + +# A unique identification for the text. +# +# epub_uid = '' + +# A tuple containing the cover image and cover page html template filenames. +# +# epub_cover = () + +# A sequence of (type, uri, title) tuples for the guide element of content.opf. +# +# epub_guide = () + +# HTML files that should be inserted before the pages created by sphinx. +# The format is a list of tuples containing the path and title. +# +# epub_pre_files = [] + +# HTML files that should be inserted after the pages created by sphinx. +# The format is a list of tuples containing the path and title. +# +# epub_post_files = [] + +# A list of files that should not be packed into the epub file. +epub_exclude_files = ['search.html'] + +# The depth of the table of contents in toc.ncx. +# +# epub_tocdepth = 3 + +# Allow duplicate toc entries. +# +# epub_tocdup = True + +# Choose between 'default' and 'includehidden'. +# +# epub_tocscope = 'default' + +# Fix unsupported image types using the Pillow. +# +# epub_fix_images = False + +# Scale large images. +# +# epub_max_image_width = 0 + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# +# epub_show_urls = 'inline' + +# If false, no index is generated. +# +# epub_use_index = True + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {'https://docs.python.org/': None} diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..354e4e1 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,22 @@ +.. PyMISP documentation master file, created by + sphinx-quickstart on Fri Aug 26 11:39:17 2016. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to PyMISP's documentation! +================================== + +Contents: + +.. toctree:: + :maxdepth: 2 + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/docs/source/modules.rst b/docs/source/modules.rst new file mode 100644 index 0000000..1bb98dd --- /dev/null +++ b/docs/source/modules.rst @@ -0,0 +1,7 @@ +pymisp +====== + +.. toctree:: + :maxdepth: 4 + + pymisp diff --git a/docs/source/pymisp.rst b/docs/source/pymisp.rst new file mode 100644 index 0000000..28ca0d9 --- /dev/null +++ b/docs/source/pymisp.rst @@ -0,0 +1,22 @@ +pymisp package +============== + +Submodules +---------- + +pymisp.api module +----------------- + +.. automodule:: pymisp.api + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: pymisp + :members: + :undoc-members: + :show-inheritance: diff --git a/pymisp/api.py b/pymisp/api.py index 8e173d5..578d9cd 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -99,7 +99,7 @@ class PyMISP(object): :param out_type: Type of object (json) NOTE: XML output isn't supported anymore, keeping the flag for compatibility reasons. :param debug: print all the messages received from the server :param proxies: Proxy dict as describes here: http://docs.python-requests.org/en/master/user/advanced/#proxies - :param proxies: Client certificate, as described there: http://docs.python-requests.org/en/master/user/advanced/#ssl-cert-verification + :param cert: Client certificate, as described there: http://docs.python-requests.org/en/master/user/advanced/#ssl-cert-verification """ # So it can may be accessed from the misp object. From a6ae9ae5f405812c74b51fa35dad858639caf6aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 26 Aug 2016 12:13:58 +0200 Subject: [PATCH 107/223] Update doc --- README.md | 27 ++++++++++++++++----------- docs/source/conf.py | 6 +++--- docs/source/index.rst | 3 +++ 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index bcbfe7a..53e40d6 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ +README +====== + + [![Build Status](https://travis-ci.org/MISP/PyMISP.svg?branch=master)](https://travis-ci.org/MISP/PyMISP) [![Coverage Status](https://coveralls.io/repos/github/MISP/PyMISP/badge.svg?branch=master)](https://coveralls.io/github/MISP/PyMISP?branch=master) @@ -12,16 +16,17 @@ PyMISP allows you to fetch events, add or update events/attributes, add or updat * [requests](http://docs.python-requests.org) ## Install from pip -~~~~ + +``` pip install pymisp -~~~~ +``` ## Install the lastest version from repo -~~~~ +``` git clone https://github.com/CIRCL/PyMISP.git && cd PyMISP python setup.py install -~~~~ +``` ## Samples and how to use PyMISP @@ -29,21 +34,21 @@ Various examples and samples scripts are in the [examples/](examples/) directory In the examples directory, you will need to change the keys.py.sample to enter your MISP url and API key. -~~~~ +``` cd examples cp keys.py.sample keys.py vim keys.py -~~~~ +``` The API key of MISP is available in the Automation section of the MISP web interface. To test if your URL and API keys are correct, you can test with examples/last.py to fetch the last 10 events published. -~~~~ +``` cd examples python last.py -l 10 -~~~~ +``` ## Documentation @@ -51,6 +56,6 @@ python last.py -l 10 Documentation can be generated with epydoc: -~~~~ - epydoc --url https://github.com/CIRCL/PyMISP --graph all --name PyMISP --pdf pymisp -o doc -~~~~ +``` +epydoc --url https://github.com/CIRCL/PyMISP --graph all --name PyMISP --pdf pymisp -o doc +``` diff --git a/docs/source/conf.py b/docs/source/conf.py index 59888b2..bc152a3 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -17,9 +17,9 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) +import os +import sys +sys.path.insert(0, os.path.abspath('.')) from recommonmark.parser import CommonMarkParser diff --git a/docs/source/index.rst b/docs/source/index.rst index 354e4e1..9a68fb7 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -11,6 +11,9 @@ Contents: .. toctree:: :maxdepth: 2 + readme + modules + Indices and tables From 2bebae7e8aa8f644b74112614fa576ffb6e76054 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 26 Aug 2016 14:01:17 +0200 Subject: [PATCH 108/223] Fix auto generation of doc --- docs/source/conf.py | 2 +- docs/source/readme.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 docs/source/readme.rst diff --git a/docs/source/conf.py b/docs/source/conf.py index bc152a3..61adec2 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -41,7 +41,7 @@ extensions = [ 'sphinx.ext.mathjax', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode', - 'sphinx.ext.githubpages', +# 'sphinx.ext.githubpages', ] # Add any paths that contain templates here, relative to this directory. diff --git a/docs/source/readme.rst b/docs/source/readme.rst new file mode 100644 index 0000000..7592303 --- /dev/null +++ b/docs/source/readme.rst @@ -0,0 +1 @@ +.. include:: ../../README.md From 77e3ad00160b16c2b7a143f555cb3dc342dc5ed0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 26 Aug 2016 14:19:19 +0200 Subject: [PATCH 109/223] Update rendering doc --- docs/source/conf.py | 7 ++++++- pymisp/api.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 61adec2..3984a79 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -41,9 +41,14 @@ extensions = [ 'sphinx.ext.mathjax', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode', -# 'sphinx.ext.githubpages', + 'sphinx.ext.githubpages', + 'sphinx.ext.napoleon', ] +napoleon_google_docstring = False +napoleon_use_param = False +napoleon_use_ivar = True + # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/pymisp/api.py b/pymisp/api.py index 578d9cd..ecd4f42 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -""" Python API using the REST interface of MISP """ +"""Python API using the REST interface of MISP""" import json import datetime From 5bf6f52301f6ea59f19a2d8c628397c4f07143f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 26 Aug 2016 17:01:32 +0200 Subject: [PATCH 110/223] Add badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 53e40d6..ed28248 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ README ====== - +[![Documentation Status](https://readthedocs.org/projects/pymisp/badge/?version=master)](http://pymisp.readthedocs.io/en/master/?badge=master) [![Build Status](https://travis-ci.org/MISP/PyMISP.svg?branch=master)](https://travis-ci.org/MISP/PyMISP) [![Coverage Status](https://coveralls.io/repos/github/MISP/PyMISP/badge.svg?branch=master)](https://coveralls.io/github/MISP/PyMISP?branch=master) From aaaab590f5bd45ad7f5dc797d3d3a08d015627ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 26 Aug 2016 18:22:41 +0200 Subject: [PATCH 111/223] Fix error flattening --- pymisp/api.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index ecd4f42..eef425c 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -166,10 +166,17 @@ class PyMISP(object): elif response.get('errors'): if isinstance(response['errors'], dict): for where, errors in response['errors'].items(): - for e in errors: - for type_e, msgs in e.items(): - for m in msgs: - messages.append('Error in {}: {}'.format(where, m)) + if isinstance(errors, dict): + for where, msg in errors.items(): + messages.append('Error in {}: {}'.format(where, msg)) + else: + for e in errors: + if isinstance(e, str): + messages.append(e) + continue + for type_e, msgs in e.items(): + for m in msgs: + messages.append('Error in {}: {}'.format(where, m)) return messages def _check_response(self, response): From 1532bc879f81175a225e841c087092b08ff46a1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 26 Aug 2016 18:23:02 +0200 Subject: [PATCH 112/223] Improve testing --- tests/test_offline.py | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/tests/test_offline.py b/tests/test_offline.py index 1a6497a..a7fbf74 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -17,6 +17,7 @@ class TestOffline(unittest.TestCase): self.domain = 'http://misp.local/' self.key = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' self.event = {'Event': json.load(open('tests/misp_event.json', 'r'))} + self.new_misp_event = {'Event': json.load(open('tests/new_misp_event.json', 'r'))} self.types = json.load(open('tests/describeTypes.json', 'r')) self.sharing_groups = json.load(open('tests/sharing_groups.json', 'r')) self.auth_error_msg = {"name": "Authentication failed. Please make sure you pass the API key of an API enabled user along in the Authorization header.", @@ -36,7 +37,7 @@ class TestOffline(unittest.TestCase): def test_getEvent(self, m): self.initURI(m) - pymisp = PyMISP(self.domain, self.key, debug=True) + pymisp = PyMISP(self.domain, self.key) e1 = pymisp.get_event(2) e2 = pymisp.get(2) self.assertEqual(e1, e2) @@ -44,7 +45,7 @@ class TestOffline(unittest.TestCase): def test_updateEvent(self, m): self.initURI(m) - pymisp = PyMISP(self.domain, self.key, debug=True) + pymisp = PyMISP(self.domain, self.key) e0 = pymisp.update_event(2, json.dumps(self.event)) e1 = pymisp.update_event(2, self.event) self.assertEqual(e0, e1) @@ -54,7 +55,7 @@ class TestOffline(unittest.TestCase): def test_deleteEvent(self, m): self.initURI(m) - pymisp = PyMISP(self.domain, self.key, debug=True) + pymisp = PyMISP(self.domain, self.key) d = pymisp.delete_event(2) self.assertEqual(d, {'message': 'Event deleted.'}) d = pymisp.delete_event(3) @@ -62,13 +63,13 @@ class TestOffline(unittest.TestCase): def test_deleteAttribute(self, m): self.initURI(m) - pymisp = PyMISP(self.domain, self.key, debug=True) + pymisp = PyMISP(self.domain, self.key) d = pymisp.delete_attribute(2) self.assertEqual(d, {'message': 'Attribute deleted.'}) def test_publish(self, m): self.initURI(m) - pymisp = PyMISP(self.domain, self.key, debug=True) + pymisp = PyMISP(self.domain, self.key) e = pymisp.publish(self.event) pub = self.event pub['Event']['published'] = True @@ -78,7 +79,7 @@ class TestOffline(unittest.TestCase): def test_getVersions(self, m): self.initURI(m) - pymisp = PyMISP(self.domain, self.key, debug=True) + pymisp = PyMISP(self.domain, self.key) api_version = pymisp.get_api_version() self.assertEqual(api_version, {'version': pm.__version__}) server_version = pymisp.get_version() @@ -86,17 +87,36 @@ class TestOffline(unittest.TestCase): def test_getSharingGroups(self, m): self.initURI(m) - pymisp = PyMISP(self.domain, self.key, debug=True) + pymisp = PyMISP(self.domain, self.key) sharing_groups = pymisp.get_sharing_groups() self.assertEqual(sharing_groups, self.sharing_groups['response'][0]) def test_auth_error(self, m): self.initURI(m) - pymisp = PyMISP(self.domain, self.key, debug=True) + pymisp = PyMISP(self.domain, self.key) error = pymisp.get(1) response = self.auth_error_msg response['errors'] = [response['message']] - print(error) + self.assertEqual(error, response) + + def test_newEvent(self, m): + error_empty_info = {'message': 'The event could not be saved.', 'name': 'Add event failed.', 'errors': {'Event': {'info': ['Info cannot be empty.']}}, 'url': '/events/add'} + error_empty_info_flatten = {'message': 'The event could not be saved.', 'name': 'Add event failed.', 'errors': ["Error in info: ['Info cannot be empty.']"], 'url': '/events/add'} + self.initURI(m) + pymisp = PyMISP(self.domain, self.key) + with self.assertRaises(pm.api.NewEventError): + pymisp.new_event() + with self.assertRaises(pm.api.NewEventError): + pymisp.new_event(0) + with self.assertRaises(pm.api.NewEventError): + pymisp.new_event(0, 1) + m.register_uri('POST', self.domain + 'events', json=error_empty_info) + response = pymisp.new_event(0, 1, 0) + self.assertEqual(response, error_empty_info_flatten) + m.register_uri('POST', self.domain + 'events', json=self.new_misp_event) + response = pymisp.new_event(0, 1, 0, "This is a test.", '2016-08-26', False) + self.assertEqual(response, self.new_misp_event) + if __name__ == '__main__': unittest.main() From 8dbeec3f9695e1821f1c600dd98f8a02e5da70d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 26 Aug 2016 18:23:20 +0200 Subject: [PATCH 113/223] Cleanup create_events --- examples/create_events.py | 7 ++----- tests/new_misp_event.json | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 tests/new_misp_event.json diff --git a/examples/create_events.py b/examples/create_events.py index 17672eb..f780511 100755 --- a/examples/create_events.py +++ b/examples/create_events.py @@ -13,7 +13,7 @@ except NameError: def init(url, key): - return PyMISP(url, key, True, 'json') + return PyMISP(url, key, True, 'json', debug=True) if __name__ == '__main__': parser = argparse.ArgumentParser(description='Create an event on MISP.') @@ -26,7 +26,4 @@ if __name__ == '__main__': misp = init(misp_url, misp_key) event = misp.new_event(args.distrib, args.threat, args.analysis, args.info) - print event - - response = misp.add_mutex(event, 'booh') - print response + print(event) diff --git a/tests/new_misp_event.json b/tests/new_misp_event.json new file mode 100644 index 0000000..88ac8f9 --- /dev/null +++ b/tests/new_misp_event.json @@ -0,0 +1,34 @@ +{ + "Event": { + "uuid": "57c06bb1-625c-4d34-9b9f-4066950d210f", + "orgc_id": "1", + "publish_timestamp": "0", + "RelatedEvent": [], + "org_id": "1", + "Org": { + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f", + "name": "CIRCL", + "id": "1" + }, + "attribute_count": null, + "distribution": "0", + "sharing_group_id": "0", + "threat_level_id": "1", + "locked": false, + "Attribute": [], + "published": false, + "ShadowAttribute": [], + "date": "2016-08-26", + "info": "This is a test", + "timestamp": "1472228273", + "Orgc": { + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f", + "name": "CIRCL", + "id": "1" + }, + "id": "594", + "proposal_email_lock": false, + "analysis": "0" + } +} + From cd25559c94eed8aeeef64562a4ce8b1e6760c62f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sat, 27 Aug 2016 18:13:15 +0200 Subject: [PATCH 114/223] Fix flattening, fix python2.7 --- pymisp/api.py | 6 +++++- tests/test_offline.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index eef425c..8b73529 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -168,7 +168,11 @@ class PyMISP(object): for where, errors in response['errors'].items(): if isinstance(errors, dict): for where, msg in errors.items(): - messages.append('Error in {}: {}'.format(where, msg)) + if isinstance(msg, list): + for m in msg: + messages.append('Error in {}: {}'.format(where, m)) + else: + messages.append('Error in {}: {}'.format(where, msg)) else: for e in errors: if isinstance(e, str): diff --git a/tests/test_offline.py b/tests/test_offline.py index a7fbf74..0dde96a 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -101,7 +101,7 @@ class TestOffline(unittest.TestCase): def test_newEvent(self, m): error_empty_info = {'message': 'The event could not be saved.', 'name': 'Add event failed.', 'errors': {'Event': {'info': ['Info cannot be empty.']}}, 'url': '/events/add'} - error_empty_info_flatten = {'message': 'The event could not be saved.', 'name': 'Add event failed.', 'errors': ["Error in info: ['Info cannot be empty.']"], 'url': '/events/add'} + error_empty_info_flatten = {u'message': u'The event could not be saved.', u'name': u'Add event failed.', u'errors': [u"Error in info: Info cannot be empty."], u'url': u'/events/add'} self.initURI(m) pymisp = PyMISP(self.domain, self.key) with self.assertRaises(pm.api.NewEventError): From 8059ead9cf451f2081f43e88e77938844cde72d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 29 Aug 2016 11:05:18 +0200 Subject: [PATCH 115/223] Bump to 2.4.51 --- pymisp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index be681dc..924282a 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,3 +1,3 @@ -__version__ = '2.4.50' +__version__ = '2.4.51' from .api import PyMISP, PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey From eb427e89c9f61c2e97e6bc8ce1de2c2b9bef5687 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Tue, 30 Aug 2016 10:42:34 +0200 Subject: [PATCH 116/223] update examples/situational-awareness/README.md --- examples/situational-awareness/README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/examples/situational-awareness/README.md b/examples/situational-awareness/README.md index 86f06bd..7512df4 100644 --- a/examples/situational-awareness/README.md +++ b/examples/situational-awareness/README.md @@ -12,8 +12,19 @@ * tag\_search.py allows research for multiple tags is possible by separating each tag by the | symbol. * Partial research is also possible with tag\_search.py. For instance, search for "ransom" will also return tags containin "ransomware". +* tags\_to\_graphs.py is a script that will generate several plots to visualise tags distribution. + * The studied _period_ can be either the 7, 28 or 360 last days + * _accuracy_ allows to get smallers splits of data instead of the default values + * _order_ define the accuracy of the curve fitting. Default value is 3 + * It will generate three plots: + * Raw datas: in plot folder, named with the name of the corresponding taxonomy + * Trend: in plot folder, named _taxonomy_\_trend. general evolution of the data (linear fitting, curve fitting at order 1) + * Curve fitting: in plotlib folder, name as the taxonomy it presents. + + :warning: These scripts are not time optimised ## Requierements * [Pygal](https://github.com/Kozea/pygal/) +* [Matplotlib](https://github.com/matplotlib/matplotlib) From d5bdb6709047d10de21bbbd25f79ff53d1294d7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Mon, 5 Sep 2016 13:41:02 +0200 Subject: [PATCH 117/223] update examples/situational-awareness/README.md --- examples/situational-awareness/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/situational-awareness/README.md b/examples/situational-awareness/README.md index 7512df4..c996498 100644 --- a/examples/situational-awareness/README.md +++ b/examples/situational-awareness/README.md @@ -16,7 +16,10 @@ * The studied _period_ can be either the 7, 28 or 360 last days * _accuracy_ allows to get smallers splits of data instead of the default values * _order_ define the accuracy of the curve fitting. Default value is 3 - * It will generate three plots: + * It will generate two plots comparing all the tags: + * tags_repartition_plot that present the raw data + * tags_repartition_trend_plot that present the general evolution for each tag + * Then each taxonomies will be represented in three plots: * Raw datas: in plot folder, named with the name of the corresponding taxonomy * Trend: in plot folder, named _taxonomy_\_trend. general evolution of the data (linear fitting, curve fitting at order 1) * Curve fitting: in plotlib folder, name as the taxonomy it presents. From fa66c77cd1ee6f8d9a88a8729d232f4594ac18de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Mon, 5 Sep 2016 14:14:29 +0200 Subject: [PATCH 118/223] add tags_to_graphs.py in ecamples/situational-awareness --- examples/situational-awareness/README.md | 8 +- examples/situational-awareness/style.css | 6 +- examples/situational-awareness/style2.css | 41 ++ .../situational-awareness/tags_to_graphs.py | 91 +++++ examples/situational-awareness/tools.py | 349 ++++++++++++++++-- 5 files changed, 451 insertions(+), 44 deletions(-) create mode 100644 examples/situational-awareness/style2.css create mode 100644 examples/situational-awareness/tags_to_graphs.py diff --git a/examples/situational-awareness/README.md b/examples/situational-awareness/README.md index c996498..d481f76 100644 --- a/examples/situational-awareness/README.md +++ b/examples/situational-awareness/README.md @@ -20,10 +20,10 @@ * tags_repartition_plot that present the raw data * tags_repartition_trend_plot that present the general evolution for each tag * Then each taxonomies will be represented in three plots: - * Raw datas: in plot folder, named with the name of the corresponding taxonomy - * Trend: in plot folder, named _taxonomy_\_trend. general evolution of the data (linear fitting, curve fitting at order 1) - * Curve fitting: in plotlib folder, name as the taxonomy it presents. - + * Raw datas: in "plot" folder, named with the name of the corresponding taxonomy + * Trend: in "plot" folder, named _taxonomy_\_trend. general evolution of the data (linear fitting, curve fitting at order 1) + * Curve fitting: in "plotlib" folder, name as the taxonomy it presents. + * In order to visualize the last plots, a html file is also generated automaticaly (might be improved in the future) :warning: These scripts are not time optimised diff --git a/examples/situational-awareness/style.css b/examples/situational-awareness/style.css index 5afdf7f..ce23448 100644 --- a/examples/situational-awareness/style.css +++ b/examples/situational-awareness/style.css @@ -29,11 +29,15 @@ table td { border-left: 1px solid #cbcbcb; border-width: 0 0 0 1px; - width: 150px; + width: 500px; margin: 0; padding: 0.5em 1em; } +.test +{ + width: 500px; +} table tr:nth-child(2n-1) td { diff --git a/examples/situational-awareness/style2.css b/examples/situational-awareness/style2.css new file mode 100644 index 0000000..6fcec41 --- /dev/null +++ b/examples/situational-awareness/style2.css @@ -0,0 +1,41 @@ +body +{ + /*font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;*/ + font-family: Consolas, "Liberation Mono", Menlo, Courier, monospace; +} + +h1 +{ + font-size: 16px; + width: 290px; + text-align:center; +} + +/*** Stats Tables ***/ + +table +{ + border-collapse: collapse; + border-spacing: 0; + table-layout: fixed; + width: 6000px; + border: 1px solid #cbcbcb; +} + +tbody +{ + font-size:12px; +} + +td +{ + border-left: 1px solid #cbcbcb; + border-width: 0 0 0 1px; + margin: 0; + padding: 0.5em 1em; +} + +table tr td:first-child +{ + font-weight: bold; +} diff --git a/examples/situational-awareness/tags_to_graphs.py b/examples/situational-awareness/tags_to_graphs.py new file mode 100644 index 0000000..7280165 --- /dev/null +++ b/examples/situational-awareness/tags_to_graphs.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pymisp import PyMISP +from keys import misp_url, misp_key, misp_verifycert +import argparse +import tools + + +def formattingDataframe(dataframe, dates, NanValue): + dataframe.reverse() + dates.reverse() + dataframe = tools.concat(dataframe) + dataframe = tools.renameColumns(dataframe, dates) + dataframe = tools.replaceNaN(dataframe, 0) + return dataframe + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Show the evolution of trend of tags.') + parser.add_argument("-p", "--period", help='Define the studied period. Can be the past year (y), month (m) or week (w). Week is the default value if no valid value is given.') + parser.add_argument("-a", "--accuracy", help='Define the accuracy of the splits on the studied period. Can be per month (m) -for year only-, week (w) -month only- or day (d). The default value is always the biggest available.') + parser.add_argument("-o", "--order", type=int, help='Define the accuracy of the curve fitting. Default value is 3') + + args = parser.parse_args() + + misp = PyMISP(misp_url, misp_key, misp_verifycert) + + if args.period == "y": + if args.accuracy == "d": + split = 360 + size = 1 + else: + split = 12 + size = 30 + last = '360d' + title = 'Tags repartition over the last 360 days' + elif args.period == "m": + if args.accuracy == "d": + split = 28 + size = 1 + else: + split = 4 + size = 7 + last = '28d' + title = 'Tags repartition over the last 28 days' + else: + split = 7 + size = 1 + last = '7d' + title = 'Tags repartition over the last 7 days' + + result = misp.download_last(last) + events = tools.eventsListBuildFromArray(result) + result = [] + dates = [] + enddate = tools.getToday() + colourDict = {} + faketag = False + + for i in range(split): + begindate = tools.getNDaysBefore(enddate, size) + dates.append(str(enddate.date())) + eventstemp = tools.selectInRange(events, begin=begindate, end=enddate) + if eventstemp is not None: + tags = tools.tagsListBuild(eventstemp) + if tags is not None: + tools.createDictTagsColour(colourDict, tags) + result.append(tools.getNbOccurenceTags(tags)) + else: + result.append(tools.createFakeEmptyTagsSeries()) + faketag = True + else: + result.append(tools.createFakeEmptyTagsSeries()) + faketag = True + enddate = begindate + + result = formattingDataframe(result, dates, 0) + if faketag: + result = tools.removeFaketagRow(result) + + taxonomies, emptyOther = tools.getTaxonomies(tools.getCopyDataframe(result)) + + + tools.tagsToLineChart(tools.getCopyDataframe(result), title, dates, colourDict) + tools.tagstrendToLineChart(tools.getCopyDataframe(result), title, dates, split, colourDict) + tools.tagsToTaxoLineChart(tools.getCopyDataframe(result), title, dates, colourDict, taxonomies, emptyOther) + tools.tagstrendToTaxoLineChart(tools.getCopyDataframe(result), title, dates, split, colourDict, taxonomies, emptyOther) + if args.order is None: + args.order = 3 + tools.tagsToPolyChart(tools.getCopyDataframe(result), split, colourDict, taxonomies, emptyOther, args.order) + tools.createVisualisation(taxonomies) diff --git a/examples/situational-awareness/tools.py b/examples/situational-awareness/tools.py index 6cff510..d4b9bea 100644 --- a/examples/situational-awareness/tools.py +++ b/examples/situational-awareness/tools.py @@ -9,8 +9,13 @@ import pandas from datetime import datetime from datetime import timedelta from dateutil.parser import parse - -# ############### Errors ################ +import numpy +from scipy import stats +from pytaxonomies import Taxonomies +import re +import matplotlib.pyplot as plt +from matplotlib import pylab +import os class DateError(Exception): @@ -20,30 +25,8 @@ class DateError(Exception): def __str__(self): return repr(self.value) -# ############### Tools ################ - - -def buildDoubleIndex(index1, index2, datatype): - it = -1 - newindex1 = [] - for index in index2: - if index == 0: - it += 1 - newindex1.append(index1[it]) - arrays = [newindex1, index2] - tuples = list(zip(*arrays)) - return pandas.MultiIndex.from_tuples(tuples, names=['event', datatype]) - - -def buildNewColumn(index2, column): - it = -1 - newcolumn = [] - for index in index2: - if index == 0: - it += 1 - newcolumn.append(column[it]) - return newcolumn +# ############### Date Tools ################ def dateInRange(datetimeTested, begin=None, end=None): if begin is None: @@ -53,10 +36,6 @@ def dateInRange(datetimeTested, begin=None, end=None): return begin <= datetimeTested <= end -def addColumn(dataframe, columnList, columnName): - dataframe.loc[:, columnName] = pandas.Series(columnList, index=dataframe.index) - - def toDatetime(date): return parse(date) @@ -86,6 +65,115 @@ def setEnddate(enddate): def getLastdate(last): return (datetime.now() - timedelta(days=int(last))).replace(hour=0, minute=0, second=0, microsecond=0) + +def getNDaysBefore(date, days): + return (date - timedelta(days=days)).replace(hour=0, minute=0, second=0, microsecond=0) + + +def getToday(): + return (datetime.now()).replace(hour=0, minute=0, second=0, microsecond=0) + + +# ############### Tools ################ + + +def getTaxonomies(dataframe): + taxonomies = Taxonomies() + taxonomies = list(taxonomies.keys()) + notInTaxo = [] + count = 0 + for taxonomy in taxonomies: + empty = True + for it in dataframe.iterrows(): + if it[0].startswith(taxonomy): + empty = False + dataframe = dataframe.drop([it[0]]) + count = count + 1 + if empty is True: + notInTaxo.append(taxonomy) + if dataframe.empty: + emptyOther = True + else: + emptyOther = False + for taxonomy in notInTaxo: + taxonomies.remove(taxonomy) + return taxonomies, emptyOther + + +def buildDoubleIndex(index1, index2, datatype): + it = -1 + newindex1 = [] + for index in index2: + if index == 0: + it += 1 + newindex1.append(index1[it]) + arrays = [newindex1, index2] + tuples = list(zip(*arrays)) + return pandas.MultiIndex.from_tuples(tuples, names=['event', datatype]) + + +def buildNewColumn(index2, column): + it = -1 + newcolumn = [] + for index in index2: + if index == 0: + it += 1 + newcolumn.append(column[it]) + return newcolumn + + +def addColumn(dataframe, columnList, columnName): + dataframe.loc[:, columnName] = pandas.Series(columnList, index=dataframe.index) + + +def concat(data): + return pandas.concat(data, axis=1) + + +def createFakeEmptyTagsSeries(): + return pandas.Series({'Faketag': 0}) + + +def removeFaketagRow(dataframe): + return dataframe.drop(['Faketag']) + + +def getCopyDataframe(dataframe): + return dataframe.copy() + + +def createDictTagsColour(colourDict, tags): + temp = tags.groupby(['name', 'colour']).count()['id'] + levels_name = temp.index.levels[0] + levels_colour = temp.index.levels[1] + labels_name = temp.index.labels[0] + labels_colour = temp.index.labels[1] + + for i in range(len(labels_name)): + colourDict[levels_name[labels_name[i]]] = levels_colour[labels_colour[i]] + + +def createTagsPlotStyle(dataframe, colourDict, taxonomy=None): + colours = [] + if taxonomy is not None: + for it in dataframe.iterrows(): + if it[0].startswith(taxonomy): + colours.append(colourDict[it[0]]) + else: + for it in dataframe.iterrows(): + colours.append(colourDict[it[0]]) + + style = Style(background='transparent', + plot_background='#eeeeee', + foreground='#111111', + foreground_strong='#111111', + foreground_subtle='#111111', + opacity='.6', + opacity_hover='.9', + transition='400ms ease-in', + colors=tuple(colours)) + return style + # ############### Formatting ################ @@ -129,15 +217,19 @@ def attributesListBuild(events): def tagsListBuild(Events): Tags = [] - for Tag in Events['Tag']: - if type(Tag) is not list: - continue - Tags.append(pandas.DataFrame(Tag)) - Tags = pandas.concat(Tags) - columnDate = buildNewColumn(Tags.index, Events['date']) - addColumn(Tags, columnDate, 'date') - index = buildDoubleIndex(Events.index, Tags.index, 'tag') - Tags = Tags.set_index(index) + if 'Tag' in Events.columns: + for Tag in Events['Tag']: + if type(Tag) is not list: + continue + Tags.append(pandas.DataFrame(Tag)) + if Tags: + Tags = pandas.concat(Tags) + columnDate = buildNewColumn(Tags.index, Events['date']) + addColumn(Tags, columnDate, 'date') + index = buildDoubleIndex(Events.index, Tags.index, 'tag') + Tags = Tags.set_index(index) + else: + Tags = None return Tags @@ -148,6 +240,8 @@ def selectInRange(Events, begin=None, end=None): inRange.append(Event.tolist()) inRange = pandas.DataFrame(inRange) temp = Events.columns.tolist() + if inRange.empty: + return None inRange.columns = temp return inRange @@ -160,6 +254,15 @@ def isTagIn(dataframe, tag): index.append(temp[i][0]) return index + +def renameColumns(dataframe, namelist): + dataframe.columns = namelist + return dataframe + + +def replaceNaN(dataframe, value): + return dataframe.fillna(value) + # ############### Basic Stats ################ @@ -212,7 +315,7 @@ def createTreemap(data, title, treename='attribute_treemap.svg', tablename='attr transition='400ms ease-in', colors=tuple(colors.values())) - treemap = pygal.Treemap(pretty_print=True, legend_at_bottom=True, style=style, explicit_size=True, width=2048, height=2048) + treemap = pygal.Treemap(pretty_print=True, legend_at_bottom=True, style=style) treemap.title = title treemap.print_values = True treemap.print_labels = True @@ -222,3 +325,171 @@ def createTreemap(data, title, treename='attribute_treemap.svg', tablename='attr createTable(colors, categ_types_hash) treemap.render_to_file(treename) + + +def tagsToLineChart(dataframe, title, dates, colourDict): + style = createTagsPlotStyle(dataframe, colourDict) + line_chart = pygal.Line(x_label_rotation=20, style=style, show_legend=False) + line_chart.title = title + line_chart.x_labels = dates + for it in dataframe.iterrows(): + line_chart.add(it[0], it[1].tolist()) + line_chart.render_to_file('tags_repartition_plot.svg') + + +def tagstrendToLineChart(dataframe, title, dates, split, colourDict): + style = createTagsPlotStyle(dataframe, colourDict) + line_chart = pygal.Line(x_label_rotation=20, style=style, show_legend=False) + line_chart.title = title + line_chart.x_labels = dates + xi = numpy.arange(split) + for it in dataframe.iterrows(): + slope, intercept, r_value, p_value, std_err = stats.linregress(xi, it[1]) + line = slope * xi + intercept + line_chart.add(it[0], line, show_dots=False) + line_chart.render_to_file('tags_repartition_trend_plot.svg') + + +def tagsToTaxoLineChart(dataframe, title, dates, colourDict, taxonomies, emptyOther): + style = createTagsPlotStyle(dataframe, colourDict) + line_chart = pygal.Line(x_label_rotation=20, style=style) + line_chart.title = title + line_chart.x_labels = dates + for taxonomy in taxonomies: + taxoStyle = createTagsPlotStyle(dataframe, colourDict, taxonomy) + taxo_line_chart = pygal.Line(x_label_rotation=20, style=taxoStyle) + taxo_line_chart.title = title + ': ' + taxonomy + taxo_line_chart.x_labels = dates + for it in dataframe.iterrows(): + if it[0].startswith(taxonomy): + taxo_line_chart.add(re.sub(taxonomy + ':', '', it[0]), it[1].tolist()) + dataframe = dataframe.drop([it[0]]) + taxo_line_chart.render_to_file('plot/' + taxonomy + '.svg') + + if not emptyOther: + taxoStyle = createTagsPlotStyle(dataframe, colourDict) + taxo_line_chart = pygal.Line(x_label_rotation=20, style=taxoStyle) + taxo_line_chart.title = title + ': other' + taxo_line_chart.x_labels = dates + for it in dataframe.iterrows(): + taxo_line_chart.add(it[0], it[1].tolist()) + taxo_line_chart.render_to_file('plot/other.svg') + + +def tagstrendToTaxoLineChart(dataframe, title, dates, split, colourDict, taxonomies, emptyOther): + style = createTagsPlotStyle(dataframe, colourDict) + line_chart = pygal.Line(x_label_rotation=20, style=style) + line_chart.title = title + line_chart.x_labels = dates + xi = numpy.arange(split) + for taxonomy in taxonomies: + taxoStyle = createTagsPlotStyle(dataframe, colourDict, taxonomy) + taxo_line_chart = pygal.Line(x_label_rotation=20, style=taxoStyle) + taxo_line_chart.title = title + ': ' + taxonomy + taxo_line_chart.x_labels = dates + for it in dataframe.iterrows(): + if it[0].startswith(taxonomy): + slope, intercept, r_value, p_value, std_err = stats.linregress(xi, it[1]) + line = slope * xi + intercept + taxo_line_chart.add(re.sub(taxonomy + ':', '', it[0]), line, show_dots=False) + dataframe = dataframe.drop([it[0]]) + taxo_line_chart.render_to_file('plot/' + taxonomy + '_trend.svg') + + if not emptyOther: + taxoStyle = createTagsPlotStyle(dataframe, colourDict) + taxo_line_chart = pygal.Line(x_label_rotation=20, style=taxoStyle) + taxo_line_chart.title = title + ': other' + taxo_line_chart.x_labels = dates + for it in dataframe.iterrows(): + slope, intercept, r_value, p_value, std_err = stats.linregress(xi, it[1]) + line = slope * xi + intercept + taxo_line_chart.add(it[0], line, show_dots=False) + taxo_line_chart.render_to_file('plot/other_trend.svg') + + +def tagsToPolyChart(dataframe, split, colourDict, taxonomies, emptyOther, order): + for taxonomy in taxonomies: + for it in dataframe.iterrows(): + if it[0].startswith(taxonomy): + points = [] + for i in range(split): + points.append((i, it[1][i])) + color = colourDict[it[0]] + label = re.sub(taxonomy + ':', '', it[0]) + points = numpy.array(points) + dataframe = dataframe.drop([it[0]]) + + # get x and y vectors + x = points[:, 0] + y = points[:, 1] + + # calculate polynomial + z = numpy.polyfit(x, y, order) + f = numpy.poly1d(z) + + # calculate new x's and y's + x_new = numpy.linspace(x[0], x[-1], 50) + y_new = f(x_new) + + plt.plot(x, y, '.', color=color) + plt.plot(x_new, y_new, color=color, label=label + 'trend') + + pylab.title('Polynomial Fit with Matplotlib: ' + taxonomy) + pylab.legend(loc='center left', bbox_to_anchor=(1, 0.5)) + ax = plt.gca() + ax.set_facecolor((0.898, 0.898, 0.898)) + box = ax.get_position() + ax.set_position([box.x0 - 0.01, box.y0, box.width * 0.78, box.height]) + fig = plt.gcf() + fig.set_size_inches(20, 15) + fig.savefig('plotlib/' + taxonomy + '.png') + fig.clf() + + if not emptyOther: + for it in dataframe.iterrows(): + points = [] + for i in range(split): + points.append((i, it[1][i])) + + color = colourDict[it[0]] + label = it[0] + points = numpy.array(points) + + # get x and y vectors + x = points[:, 0] + y = points[:, 1] + + # calculate polynomial + z = numpy.polyfit(x, y, order) + f = numpy.poly1d(z) + + # calculate new x's and y's + x_new = numpy.linspace(x[0], x[-1], 50) + y_new = f(x_new) + + plt.plot(x, y, '.', color=color, label=label) + plt.plot(x_new, y_new, color=color, label=label + 'trend') + + pylab.title('Polynomial Fit with Matplotlib: other') + pylab.legend(loc='center left', bbox_to_anchor=(1, 0.5)) + ax = plt.gca() + ax.set_facecolor((0.898, 0.898, 0.898)) + box = ax.get_position() + ax.set_position([box.x0 - 0.01, box.y0, box.width * 0.78, box.height]) + fig = plt.gcf() + fig.set_size_inches(20, 15) + fig.savefig('plotlib/other.png') + + +def createVisualisation(taxonomies): + chain = '\n\n\t\n\t\t\n\t\n\t' + chain = chain + '' + for taxonomy in taxonomies: + chain = chain + '\n' + + chain = chain + '\n' + chain = chain + '
graph
graph
' + chain = chain + '\n\t\n' + + with open('test_tags_trend.html', 'w') as target: + target.write(chain) From 84eb40e42b2b8c41c1c780d3bf1c0512a29bd671 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Mon, 12 Sep 2016 11:32:04 +0200 Subject: [PATCH 119/223] Add some examples --- examples/sharing_groups.py | 25 +++++++++++++++++++++++++ examples/tagstatistics.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 examples/sharing_groups.py create mode 100644 examples/tagstatistics.py diff --git a/examples/sharing_groups.py b/examples/sharing_groups.py new file mode 100644 index 0000000..3bf4fa9 --- /dev/null +++ b/examples/sharing_groups.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pymisp import PyMISP +from keys import misp_url, misp_key +import argparse + +# For python2 & 3 compat, a bit dirty, but it seems to be the least bad one +try: + input = raw_input +except NameError: + pass + + +def init(url, key): + return PyMISP(url, key, True, 'json') + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Get a list of the sharing groups from the MISP instance.') + + misp = init(misp_url, misp_key) + + sharing_groups = misp.get_sharing_groups() + print sharing_groups + diff --git a/examples/tagstatistics.py b/examples/tagstatistics.py new file mode 100644 index 0000000..9357c8e --- /dev/null +++ b/examples/tagstatistics.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pymisp import PyMISP +from keys import misp_url, misp_key, misp_verifycert +import argparse +import json + +# For python2 & 3 compat, a bit dirty, but it seems to be the least bad one +try: + input = raw_input +except NameError: + pass + + +def init(url, key): + return PyMISP(url, key, misp_verifycert, 'json') + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Get statistics from tags.') + parser.add_argument("-p", "--percentage", help="An optional field, if set, it will return the results in percentages, otherwise it returns exact count.") + parser.add_argument("-n", "--namesort", help="An optional field, if set, values are sort by the namespace, otherwise the sorting will happen on the value.") + args = parser.parse_args() + + misp = init(misp_url, misp_key) + + stats = misp.get_tags_statistics(args.percentage, args.namesort) + print json.dumps(stats) From bf5793992b6ab82bf448d409593e5da39c37ee36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 12 Sep 2016 12:53:58 +0200 Subject: [PATCH 120/223] Fix examples after removal of MISP XML support --- examples/copy_list.py | 21 +++------- examples/feed-generator/generate.py | 16 ++++---- examples/get_network_activity.py | 59 +++++++++++++---------------- examples/ioc-2-misp/ioc2misp.py | 3 +- examples/tagstatistics.py | 2 +- pymisp/api.py | 2 +- 6 files changed, 43 insertions(+), 60 deletions(-) diff --git a/examples/copy_list.py b/examples/copy_list.py index bd60e5b..1e74ccd 100755 --- a/examples/copy_list.py +++ b/examples/copy_list.py @@ -27,25 +27,14 @@ def init(cert_to_priv=True): destination = PyMISP(url_cert, cert, cert_cert, 'xml') -def _to_utf8(request): - to_return = None - if 'json' in request.headers['content-type']: - to_return = request.json() - else: - to_return = request.text.encode('utf-8') - return to_return - - def copy_event(event_id): - r_src = source.get_event(event_id) - to_send = _to_utf8(r_src) - return destination.add_event(to_send) + e = source.get_event(event_id) + return destination.add_event(e) def update_event(event_id, event_to_update): - r_src = source.get_event(event_id) - to_send = _to_utf8(r_src) - return destination.update_event(event_to_update, to_send) + e = source.get_event(event_id) + return destination.update_event(event_to_update, e) def list_copy(filename): @@ -83,7 +72,7 @@ def copy(eventid): def export_our_org(): circl = source.search(org='CIRCL') - return _to_utf8(circl) + return circl if __name__ == '__main__': import argparse diff --git a/examples/feed-generator/generate.py b/examples/feed-generator/generate.py index e980ff0..2188d2a 100755 --- a/examples/feed-generator/generate.py +++ b/examples/feed-generator/generate.py @@ -34,13 +34,13 @@ def init(): valid_attribute_distributions = valid_attribute_distribution_levels except: valid_attribute_distributions = ['0', '1', '2', '3', '4', '5'] - return PyMISP(url, key, ssl, 'json') + return PyMISP(url, key, ssl) def saveEvent(misp, uuid): event = misp.get_event(uuid) - if not event.json().get('Event'): - print('Error while fetching event: {}'.format(event.json()['message'])) + if not event.get('Event'): + print('Error while fetching event: {}'.format(event['message'])) sys.exit('Could not create file for event ' + uuid + '.') event = __cleanUpEvent(event) event = json.dumps(event) @@ -50,7 +50,7 @@ def saveEvent(misp, uuid): def __cleanUpEvent(event): - temp = event.json() + temp = event event = {'Event': {}} __cleanupEventFields(event, temp) __cleanupEventObjects(event, temp) @@ -120,10 +120,12 @@ def __addEventToManifest(event): if __name__ == '__main__': misp = init() - result = misp.get_index(None, filters) try: - events = result.json() - except: + r = misp.get_index(filters) + events = r['response'] + print(events[0]) + except Exception as e: + print(e) sys.exit("Invalid response received from MISP.") if len(events) == 0: sys.exit("No events returned.") diff --git a/examples/get_network_activity.py b/examples/get_network_activity.py index 433a9d8..03a1c1c 100755 --- a/examples/get_network_activity.py +++ b/examples/get_network_activity.py @@ -48,41 +48,34 @@ def get_event(event_id): event_id = int(event_id) if event_id > 0: - event = source.get_event(event_id) - if event.status_code == 200: + event_json = source.get_event(event_id) + event_core = event_json["Event"] + # event_threatlevel_id = event_core["threat_level_id"] - try: - event_json = event.json() - except: - return False + # attribute_count = event_core["attribute_count"] + attribute = event_core["Attribute"] - event_core = event_json["Event"] - # event_threatlevel_id = event_core["threat_level_id"] + for attribute in event_core["Attribute"]: + if app_ids_only and not attribute["to_ids"]: + continue - # attribute_count = event_core["attribute_count"] - attribute = event_core["Attribute"] - - for attribute in event_core["Attribute"]: - if app_ids_only and not attribute["to_ids"]: - continue - - value = attribute["value"] - title = event_core["info"] - if app_netflow: - app_printcomment = False - if attribute["type"] == "ip-dst" and app_ip_dst: - network_ip_dst.append([build_entry(value, event_id, title, "ip-dst")]) + value = attribute["value"] + title = event_core["info"] + if app_netflow: + app_printcomment = False + if attribute["type"] == "ip-dst" and app_ip_dst: + network_ip_dst.append([build_entry(value, event_id, title, "ip-dst")]) + else: + if attribute["type"] == "ip-src" and app_ip_src: + network_ip_src.append([build_entry(value, event_id, title, "ip-src")]) + elif attribute["type"] == "ip-dst" and app_ip_dst: + network_ip_dst.append([build_entry(value, event_id, title, "ip-dst")]) + elif attribute["type"] == "domain" and app_domain: + network_domain.append([build_entry(value, event_id, title, "domain")]) + elif attribute["type"] == "hostname" and app_hostname: + network_hostname.append([build_entry(value, event_id, title, "hostname")]) else: - if attribute["type"] == "ip-src" and app_ip_src: - network_ip_src.append([build_entry(value, event_id, title, "ip-src")]) - elif attribute["type"] == "ip-dst" and app_ip_dst: - network_ip_dst.append([build_entry(value, event_id, title, "ip-dst")]) - elif attribute["type"] == "domain" and app_domain: - network_domain.append([build_entry(value, event_id, title, "domain")]) - elif attribute["type"] == "hostname" and app_hostname: - network_hostname.append([build_entry(value, event_id, title, "hostname")]) - else: - continue + continue else: print("Not a valid ID") return @@ -121,8 +114,8 @@ def print_events(): if firsthost: firsthost = False else: - print " or " - print "host %s" % ip[0] + print(" or ") + print("host %s" % ip[0]) else: if app_ip_src: for ip in network_ip_src: diff --git a/examples/ioc-2-misp/ioc2misp.py b/examples/ioc-2-misp/ioc2misp.py index 5becc74..a7bc458 100755 --- a/examples/ioc-2-misp/ioc2misp.py +++ b/examples/ioc-2-misp/ioc2misp.py @@ -228,8 +228,7 @@ def push_event_to_misp(jsonEvent): #################### # upload json event - r = misp.add_event(jsonEvent) - event = r.json() + event = misp.add_event(jsonEvent) # save event id for file upload and tagg iocDescriptions["misp_event_id"] = event["Event"]["id"] diff --git a/examples/tagstatistics.py b/examples/tagstatistics.py index 9357c8e..9e5a958 100644 --- a/examples/tagstatistics.py +++ b/examples/tagstatistics.py @@ -25,4 +25,4 @@ if __name__ == '__main__': misp = init(misp_url, misp_key) stats = misp.get_tags_statistics(args.percentage, args.namesort) - print json.dumps(stats) + print(json.dumps(stats)) diff --git a/pymisp/api.py b/pymisp/api.py index 8b73529..6c80b62 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1138,7 +1138,7 @@ class PyMISP(object): else: name_sort = 'false' url = urljoin(self.root_url, 'tags/tagStatistics/{}/{}'.format(percentage, name_sort)) - response = session.get(url).json() + response = session.get(url) return self._check_response(response) # ############## Sightings ################## From aef6bbc32db5f2eec042f2776ff9f91f59a99358 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 12 Sep 2016 13:09:26 +0200 Subject: [PATCH 121/223] Version bump --- pymisp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 924282a..36eb6d2 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,3 +1,3 @@ -__version__ = '2.4.51' +__version__ = '2.4.51.1' from .api import PyMISP, PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey From e70cc7a985b84605145cbe7a760304c86bcdef1b Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 12 Sep 2016 13:45:37 +0200 Subject: [PATCH 122/223] Toggle flag instead of value --- examples/tagstatistics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/tagstatistics.py b/examples/tagstatistics.py index 9e5a958..4f9fe76 100644 --- a/examples/tagstatistics.py +++ b/examples/tagstatistics.py @@ -18,8 +18,8 @@ def init(url, key): if __name__ == '__main__': parser = argparse.ArgumentParser(description='Get statistics from tags.') - parser.add_argument("-p", "--percentage", help="An optional field, if set, it will return the results in percentages, otherwise it returns exact count.") - parser.add_argument("-n", "--namesort", help="An optional field, if set, values are sort by the namespace, otherwise the sorting will happen on the value.") + parser.add_argument("-p", "--percentage", action='store_true', default=None, help="An optional field, if set, it will return the results in percentages, otherwise it returns exact count.") + parser.add_argument("-n", "--namesort", action='store_true', default=None, help="An optional field, if set, values are sort by the namespace, otherwise the sorting will happen on the value.") args = parser.parse_args() misp = init(misp_url, misp_key) From f6cf9d9150c54d12873487f62a0dc3d4ed332a1c Mon Sep 17 00:00:00 2001 From: Hannah Ward Date: Tue, 13 Sep 2016 13:03:22 +0100 Subject: [PATCH 123/223] Fixed search_index --- pymisp/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/api.py b/pymisp/api.py index 6c80b62..c44577b 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -867,7 +867,7 @@ class PyMISP(object): if allowed[rule] is not None: if not isinstance(allowed[rule], list): allowed[rule] = [allowed[rule]] - allowed[rule] = map(str, allowed[rule]) + allowed[rule] = [x for x in map(str, allowed[rule])] if rule in rule_levels: if not set(allowed[rule]).issubset(rule_levels[rule]): raise SearchError('Values in your {} are invalid, has to be in {}'.format(rule, ', '.join(str(x) for x in rule_levels[rule]))) From cf257493f7cf377a734d91477498b6175dd50f26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 26 Sep 2016 00:26:09 +0200 Subject: [PATCH 124/223] First batch of changes, will be squashed --- pymisp/__init__.py | 3 +- pymisp/api.py | 147 +++++++----------------------- pymisp/exceptions.py | 31 +++++++ pymisp/mispevent.py | 211 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 279 insertions(+), 113 deletions(-) create mode 100644 pymisp/exceptions.py create mode 100644 pymisp/mispevent.py diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 36eb6d2..70200cc 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,3 +1,4 @@ __version__ = '2.4.51.1' -from .api import PyMISP, PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey +from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey +from .api import PyMISP diff --git a/pymisp/api.py b/pymisp/api.py index c44577b..b20deaf 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -15,8 +15,6 @@ except ImportError: from urlparse import urljoin from io import BytesIO import zipfile -import warnings -import functools try: import requests @@ -25,6 +23,8 @@ except ImportError: HAVE_REQUESTS = False from . import __version__ +from .exceptions import PyMISPError, NewEventError, NewAttributeError, SearchError, MissingDependency, NoURL, NoKey +from .mispevent import MISPEvent, MISPAttribute # Least dirty way to support python 2 and 3 try: @@ -56,36 +56,6 @@ class analysis(object): completed = 2 -class PyMISPError(Exception): - def __init__(self, message): - super(PyMISPError, self).__init__(message) - self.message = message - - -class NewEventError(PyMISPError): - pass - - -class NewAttributeError(PyMISPError): - pass - - -class SearchError(PyMISPError): - pass - - -class MissingDependency(PyMISPError): - pass - - -class NoURL(PyMISPError): - pass - - -class NoKey(PyMISPError): - pass - - class PyMISP(object): """ Python API for MISP @@ -102,6 +72,9 @@ class PyMISP(object): :param cert: Client certificate, as described there: http://docs.python-requests.org/en/master/user/advanced/#ssl-cert-verification """ +# TODO: interactive script to create a MISP event from scratch +# TODO: a parser to verify the validity of an event + # So it can may be accessed from the misp object. distributions = distributions threat_level = threat_level @@ -138,6 +111,11 @@ class PyMISP(object): self.categories = self.describe_types['result']['categories'] self.types = self.describe_types['result']['types'] self.category_type_mapping = self.describe_types['result']['category_type_mappings'] + # New in 2.5.52 + if self.describe_types['result'].get('sane_defaults'): + self.sane_default = self.describe_types['result']['sane_defaults'] + else: + self.sane_default = {} def __prepare_session(self, output='json'): """ @@ -311,53 +289,16 @@ class PyMISP(object): # ############################################## def _prepare_full_event(self, distribution, threat_level_id, analysis, info, date=None, published=False): - to_return = {'Event': {}} - # Setup details of a new event - if distribution not in [0, 1, 2, 3]: - raise NewEventError('{} is invalid, the distribution has to be in 0, 1, 2, 3'.format(distribution)) - if threat_level_id not in [1, 2, 3, 4]: - raise NewEventError('{} is invalid, the threat_level_id has to be in 1, 2, 3, 4'.format(threat_level_id)) - if analysis not in [0, 1, 2]: - raise NewEventError('{} is invalid, the analysis has to be in 0, 1, 2'.format(analysis)) - if date is None: - date = datetime.date.today().isoformat() - if published not in [True, False]: - raise NewEventError('{} is invalid, published has to be True or False'.format(published)) - to_return['Event'] = {'distribution': distribution, 'info': info, 'date': date, 'published': published, - 'threat_level_id': threat_level_id, 'analysis': analysis} - return to_return + misp_event = MISPEvent(self.describe_types['result']) + misp_event.set_values(info, distribution, threat_level_id, analysis, date) + if published: + misp_event.publish() + return misp_event.dump() - def _prepare_full_attribute(self, category, type_value, value, to_ids, comment=None, distribution=None): - to_return = {} - if category not in self.categories: - raise NewAttributeError('{} is invalid, category has to be in {}'.format(category, (', '.join(self.categories)))) - - if type_value not in self.types: - raise NewAttributeError('{} is invalid, type_value has to be in {}'.format(type_value, (', '.join(self.types)))) - - if type_value not in self.category_type_mapping[category]: - raise NewAttributeError('{} and {} is an invalid combinaison, type_value for this category has to be in {}'.format(type_value, category, (', '.join(self.category_type_mapping[category])))) - to_return['type'] = type_value - to_return['category'] = category - - if to_ids not in [True, False]: - raise NewAttributeError('{} is invalid, to_ids has to be True or False'.format(to_ids)) - to_return['to_ids'] = to_ids - - if distribution is not None: - distribution = int(distribution) - # If None: take the default value of the event - if distribution not in [None, 0, 1, 2, 3, 5]: - raise NewAttributeError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 5 or None'.format(distribution)) - if distribution is not None: - to_return['distribution'] = distribution - - to_return['value'] = value - - if comment is not None: - to_return['comment'] = comment - - return to_return + def _prepare_full_attribute(self, category, type_value, value, to_ids, comment=None, distribution=5): + misp_attribute = MISPAttribute(self.categories, self.types, self.category_type_mapping) + misp_attribute.set_values(type_value, value, category, to_ids, comment, distribution) + return misp_attribute.dump() def _prepare_update(self, event): # Cleanup the received event to make it publishable @@ -381,30 +322,30 @@ class PyMISP(object): # ########## Helpers ########## def get(self, eid): - response = self.get_event(int(eid)) - return response + return self.get_event(eid) def get_stix(self, **kwargs): - response = self.get_stix_event(**kwargs) - return response + return self.get_stix_event(**kwargs) def update(self, event): eid = event['Event']['id'] - response = self.update_event(eid, event) - return response - - def new_event(self, distribution=None, threat_level_id=None, analysis=None, info=None, date=None, published=False): - data = self._prepare_full_event(distribution, threat_level_id, analysis, info, date, published) - response = self.add_event(data) - return response + return self.update_event(eid, event) def publish(self, event): if event['Event']['published']: return {'error': 'Already published'} event = self._prepare_update(event) event['Event']['published'] = True - response = self.update_event(event['Event']['id'], event) - return response + return self.update_event(event['Event']['id'], event) + + def change_threat_level(self, event, threat_level_id): + event['Event']['threat_level_id'] = threat_level_id + self._prepare_update(event) + return self.update_event(event['Event']['id'], event) + + def new_event(self, distribution=None, threat_level_id=None, analysis=None, info=None, date=None, published=False): + data = self._prepare_full_event(distribution, threat_level_id, analysis, info, date, published) + return self.add_event(data) def add_tag(self, event, tag): session = self.__prepare_session() @@ -418,12 +359,6 @@ class PyMISP(object): response = session.post(urljoin(self.root_url, 'events/removeTag'), data=json.dumps(to_post)) return self._check_response(response) - def change_threat_level(self, event, threat_level_id): - event['Event']['threat_level_id'] = threat_level_id - self._prepare_update(event) - response = self.update_event(event['Event']['id'], event) - return response - # ##### File attributes ##### def _send_attributes(self, event, attributes, proposal=False): @@ -708,21 +643,9 @@ class PyMISP(object): # ######### Upload samples through the API ######### # ################################################## - def _create_event(self, distribution, threat_level_id, analysis, info): - # Setup details of a new event - if distribution not in [0, 1, 2, 3]: - raise NewEventError('{} is invalid, the distribution has to be in 0, 1, 2, 3'.format(distribution)) - if threat_level_id not in [1, 2, 3, 4]: - raise NewEventError('{} is invalid, the threat_level_id has to be in 1, 2, 3, 4'.format(threat_level_id)) - if analysis not in [0, 1, 2]: - raise NewEventError('{} is invalid, the analysis has to be in 0, 1, 2'.format(analysis)) - return {'distribution': int(distribution), 'info': info, - 'threat_level_id': int(threat_level_id), 'analysis': analysis} - def prepare_attribute(self, event_id, distribution, to_ids, category, comment, info, analysis, threat_level_id): to_post = {'request': {}} - authorized_categs = ['Payload delivery', 'Artifacts dropped', 'Payload installation', 'External analysis', 'Network activity', 'Antivirus detection'] if event_id is not None: try: @@ -731,7 +654,7 @@ class PyMISP(object): pass if not isinstance(event_id, int): # New event - to_post['request'] = self._create_event(distribution, threat_level_id, analysis, info) + to_post['request'] = self._prepare_full_event(distribution, threat_level_id, analysis, info) else: to_post['request']['event_id'] = int(event_id) @@ -739,8 +662,8 @@ class PyMISP(object): raise NewAttributeError('{} is invalid, to_ids has to be True or False'.format(to_ids)) to_post['request']['to_ids'] = to_ids - if category not in authorized_categs: - raise NewAttributeError('{} is invalid, category has to be in {}'.format(category, (', '.join(authorized_categs)))) + if category not in self.categories: + raise NewAttributeError('{} is invalid, category has to be in {}'.format(category, (', '.join(self.categories)))) to_post['request']['category'] = category to_post['request']['comment'] = comment diff --git a/pymisp/exceptions.py b/pymisp/exceptions.py new file mode 100644 index 0000000..f4db340 --- /dev/null +++ b/pymisp/exceptions.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +class PyMISPError(Exception): + def __init__(self, message): + super(PyMISPError, self).__init__(message) + self.message = message + + +class NewEventError(PyMISPError): + pass + + +class NewAttributeError(PyMISPError): + pass + + +class SearchError(PyMISPError): + pass + + +class MissingDependency(PyMISPError): + pass + + +class NoURL(PyMISPError): + pass + + +class NoKey(PyMISPError): + pass diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py new file mode 100644 index 0000000..2d181b4 --- /dev/null +++ b/pymisp/mispevent.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import datetime +import time +import json + +from .exceptions import PyMISPError, NewEventError, NewAttributeError + + +class MISPAttribute(object): + + def __init__(self, categories, types, category_type_mapping): + self.categories = categories + self.types = types + self.category_type_mapping = category_type_mapping + self.new = True + + # Default values + self.category = None + self.type = None + self.value = None + self.to_ids = False + self.comment = '' + self.distribution = 5 + + def set_values(self, type_value, value, category, to_ids, comment, distribution): + self._validate(type_value, value, category, to_ids, comment, distribution) + self.type = type_value + self.value = value + self.category = category + self.to_ids = to_ids + self.comment = comment + self.distribution = distribution + + def set_values_existing_attribute(self, attribute_id, uuid, timestamp, sharing_group_id, deleted, SharingGroup, ShadowAttribute): + self.new = False + self.id = int(attribute_id) + self.uuid = uuid + self.timestamp = datetime.datetime.fromtimestamp(timestamp) + self.sharing_group_id = int(sharing_group_id) + self.deleted = deleted + self.SharingGroup = SharingGroup + self.ShadowAttribute = ShadowAttribute + + def _validate(self, type_value, value, category, to_ids, comment, distribution): + if category not in self.categories: + raise NewAttributeError('{} is invalid, category has to be in {}'.format(category, (', '.join(self.categories)))) + if type_value not in self.types: + raise NewAttributeError('{} is invalid, type_value has to be in {}'.format(type_value, (', '.join(self.types)))) + if type_value not in self.category_type_mapping[category]: + raise NewAttributeError('{} and {} is an invalid combinaison, type_value for this category has to be in {}'.capitalizeformat(type_value, category, (', '.join(self.category_type_mapping[category])))) + if to_ids not in [True, False]: + raise NewAttributeError('{} is invalid, to_ids has to be True or False'.format(to_ids)) + if distribution not in [0, 1, 2, 3, 5]: + raise NewAttributeError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 5'.format(distribution)) + + def dump(self): + to_return = {'type': self.type, 'category': self.category, 'to_ids': self.to_ids, + 'distribution': self.distribution, 'value': self.value, + 'comment': self.comment} + if not self.new: + to_return.update( + {'id': self.id, 'uuid': self.uuid, + 'timestamp': int(time.mktime(self.timestamp.timetuple())), + 'sharing_group_id': self.sharing_group_id, 'deleted': self.deleted, + 'SharingGroup': self.SharingGroup, 'ShadowAttribute': self.ShadowAttribute}) + return to_return + + +class MISPEvent(object): + + def __init__(self, describe_types): + self.categories = describe_types['categories'] + self.types = describe_types['types'] + self.category_type_mapping = describe_types['category_type_mappings'] + self.sane_default = describe_types['sane_defaults'] + self.new = True + self.dump_full = False + + # Default values + self.distribution = 3 + self.threat_level_id = 2 + self.analysis = 0 + self.info = '' + self.published = False + self.date = datetime.date.today() + self.attributes = [] + + def _validate(self, distribution, threat_level_id, analysis): + if distribution not in [0, 1, 2, 3]: + raise NewEventError('{} is invalid, the distribution has to be in 0, 1, 2, 3'.format(distribution)) + if threat_level_id not in [1, 2, 3, 4]: + raise NewEventError('{} is invalid, the threat_level has to be in 1, 2, 3, 4'.format(threat_level_id)) + if analysis not in [0, 1, 2]: + raise NewEventError('{} is invalid, the analysis has to be in 0, 1, 2'.format(analysis)) + + def load(self, json_event): + self.new = False + self.dump_full = True + loaded = json.loads(json_event) + if loaded.get('response'): + e = loaded.get('response')[0].get('Event') + else: + e = loaded.get('Event') + if not e: + raise PyMISPError('Invalid event') + try: + date = datetime.date(*map(int, e['date'].split('-'))) + except: + raise NewEventError('{} is an invalid date.'.format(e['date'])) + self.set_values(e['info'], int(e['distribution']), int(e['threat_level_id']), int(e['analysis']), date) + if e['published']: + self.publish() + self.set_values_existing_event( + e['id'], e['orgc_id'], e['org_id'], e['uuid'], + e['attribute_count'], e['proposal_email_lock'], e['locked'], + e['publish_timestamp'], e['sharing_group_id'], e['Org'], e['Orgc'], + e['ShadowAttribute'], e['RelatedEvent']) + self.attributes = [] + for a in e['Attribute']: + attribute = MISPAttribute(self.categories, self.types, self.category_type_mapping) + attribute.set_values(a['type'], a['value'], a['category'], a['to_ids'], + a['comment'], int(a['distribution'])) + attribute.set_values_existing_attribute(a['id'], a['uuid'], a['timestamp'], + a['sharing_group_id'], a['deleted'], + a['SharingGroup'], a['ShadowAttribute']) + self.attributes.append(attribute) + + def dump(self): + to_return = {'Event': {}} + to_return['Event'] = {'distribution': self.distribution, 'info': self.info, + 'date': self.date.isoformat(), 'published': self.published, + 'threat_level_id': self.threat_level_id, + 'analysis': self.analysis, 'Attribute': []} + if not self.new: + to_return['Event'].update( + {'id': self.id, 'orgc_id': self.orgc_id, 'org_id': self.org_id, + 'uuid': self.uuid, 'sharing_group_id': self.sharing_group_id}) + if self.dump_full: + to_return['Event'].update( + {'locked': self.locked, 'attribute_count': self.attribute_count, + 'RelatedEvent': self.RelatedEvent, 'Orgc': self.Orgc, + 'ShadowAttribute': self.ShadowAttribute, 'Org': self.Org, + 'proposal_email_lock': self.proposal_email_lock, + 'publish_timestamp': int(time.mktime(self.publish_timestamp.timetuple()))}) + to_return['Event']['Attribute'] = [a.dump() for a in self.attributes] + return json.dumps(to_return) + + def set_values(self, info, distribution=3, threat_level_id=2, analysis=0, date=None): + self._validate(distribution, threat_level_id, analysis) + self.info = info + self.distribution = distribution + self.threat_level_id = threat_level_id + self.analysis = analysis + if not date: + self.date = datetime.date.today() + else: + self.date = date + + def set_values_existing_event(self, event_id, orgc_id, org_id, uuid, attribute_count, + proposal_email_lock, locked, publish_timestamp, + sharing_group_id, Org, Orgc, ShadowAttribute, + RelatedEvent): + self.id = int(event_id) + self.orgc_id = int(orgc_id) + self.org_id = int(org_id) + self.uuid = uuid + self.attribute_count = int(attribute_count) + self.proposal_email_lock = proposal_email_lock + self.locked = locked + self.publish_timestamp = datetime.datetime.fromtimestamp(publish_timestamp) + self.sharing_group_id = int(sharing_group_id) + self.Org = Org + self.Orgc = Orgc + self.ShadowAttribute = ShadowAttribute + self.RelatedEvent = RelatedEvent + + def publish(self): + self.publish = True + + def unpublish(self): + self.publish = False + + def prepare_for_update(self): + self.unpublish() + self.dump_full = False + + def add_attribute(self, type_value, value, **kwargs): + if not self.sane_default.get(type_value): + raise NewAttributeError("{} is an invalid type. Can only be one of the following: {}".format(type_value, ', '.join(self.types))) + defaults = self.sane_default[type_value] + if kwargs.get('category'): + category = kwargs.get('category') + else: + category = defaults['default_category'] + if kwargs.get('to_ids'): + to_ids = bool(int(kwargs.get('to_ids'))) + else: + to_ids = bool(int(defaults['to_ids'])) + if kwargs.get('comment'): + comment = kwargs.get('comment') + else: + comment = None + if kwargs.get('distribution'): + distribution = int(kwargs.get('distribution')) + else: + distribution = 5 + attribute = MISPAttribute(self.categories, self.types, self.category_type_mapping) + attribute.set_values(type_value, value, category, to_ids, comment, distribution) + self.attributes.append(attribute) From e0359229492f6256c7b15b38ea0e7de09ec872e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 27 Sep 2016 19:47:22 +0200 Subject: [PATCH 125/223] Getting closed to a full support of a misp event as a Python Object --- MANIFEST.in | 1 + pymisp/__init__.py | 1 + pymisp/api.py | 51 ++--- pymisp/data/describeTypes.json | 1 + pymisp/data/schema-lax.json | 322 +++++++++++++++++++++++++++++ pymisp/data/schema.json | 327 +++++++++++++++++++++++++++++ pymisp/mispevent.py | 366 +++++++++++++++++++++----------- setup.py | 4 +- tests/describeTypes.json | 368 --------------------------------- tests/test_offline.py | 25 ++- 10 files changed, 936 insertions(+), 530 deletions(-) create mode 100644 MANIFEST.in create mode 100644 pymisp/data/describeTypes.json create mode 100644 pymisp/data/schema-lax.json create mode 100644 pymisp/data/schema.json delete mode 100644 tests/describeTypes.json diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..d1cf49c --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include pymisp/data/* diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 70200cc..7a17886 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -2,3 +2,4 @@ __version__ = '2.4.51.1' from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey from .api import PyMISP +from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull diff --git a/pymisp/api.py b/pymisp/api.py index b20deaf..6fc41e1 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -23,8 +23,8 @@ except ImportError: HAVE_REQUESTS = False from . import __version__ -from .exceptions import PyMISPError, NewEventError, NewAttributeError, SearchError, MissingDependency, NoURL, NoKey -from .mispevent import MISPEvent, MISPAttribute +from .exceptions import PyMISPError, NewAttributeError, SearchError, MissingDependency, NoURL, NoKey +from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate # Least dirty way to support python 2 and 3 try: @@ -290,30 +290,17 @@ class PyMISP(object): def _prepare_full_event(self, distribution, threat_level_id, analysis, info, date=None, published=False): misp_event = MISPEvent(self.describe_types['result']) - misp_event.set_values(info, distribution, threat_level_id, analysis, date) + misp_event.set_all_values(info=info, distribution=distribution, threat_level_id=threat_level_id, + analysis=analysis, date=date) if published: misp_event.publish() - return misp_event.dump() + return json.dumps(misp_event, cls=EncodeUpdate) def _prepare_full_attribute(self, category, type_value, value, to_ids, comment=None, distribution=5): misp_attribute = MISPAttribute(self.categories, self.types, self.category_type_mapping) - misp_attribute.set_values(type_value, value, category, to_ids, comment, distribution) - return misp_attribute.dump() - - def _prepare_update(self, event): - # Cleanup the received event to make it publishable - event['Event'].pop('locked', None) - event['Event'].pop('attribute_count', None) - event['Event'].pop('RelatedEvent', None) - event['Event'].pop('orgc', None) - event['Event'].pop('ShadowAttribute', None) - event['Event'].pop('org', None) - event['Event'].pop('proposal_email_lock', None) - event['Event'].pop('publish_timestamp', None) - event['Event'].pop('published', None) - event['Event'].pop('timestamp', None) - event['Event']['id'] = int(event['Event']['id']) - return event + misp_attribute.set_all_values(type=type_value, value=value, category=category, + to_ids=to_ids, comment=comment, distribution=distribution) + return json.dumps(misp_attribute, cls=EncodeUpdate) def _one_or_more(self, value): """Returns a list/tuple of one or more items, regardless of input.""" @@ -334,14 +321,16 @@ class PyMISP(object): def publish(self, event): if event['Event']['published']: return {'error': 'Already published'} - event = self._prepare_update(event) - event['Event']['published'] = True - return self.update_event(event['Event']['id'], event) + e = MISPEvent(self.describe_types['result']) + e.load(event) + e.publish() + return self.update_event(event['Event']['id'], json.dumps(e, cls=EncodeUpdate)) def change_threat_level(self, event, threat_level_id): - event['Event']['threat_level_id'] = threat_level_id - self._prepare_update(event) - return self.update_event(event['Event']['id'], event) + e = MISPEvent(self.describe_types['result']) + e.load(event) + e.threat_level_id = threat_level_id + return self.update_event(event['Event']['id'], json.dumps(e, cls=EncodeUpdate)) def new_event(self, distribution=None, threat_level_id=None, analysis=None, info=None, date=None, published=False): data = self._prepare_full_event(distribution, threat_level_id, analysis, info, date, published) @@ -365,12 +354,12 @@ class PyMISP(object): if proposal: response = self.proposal_add(event['Event']['id'], attributes) else: - event = self._prepare_update(event) - for a in attributes: + e = MISPEvent(self.describe_types['result']) + e.load(event) + for a in e.attributes: if a.get('distribution') is None: a['distribution'] = 5 - event['Event']['Attribute'] = attributes - response = self.update_event(event['Event']['id'], event) + response = self.update_event(event['Event']['id'], json.dumps(e, cls=EncodeUpdate)) return response def add_named_attribute(self, event, category, type_value, value, to_ids=False, comment=None, distribution=None, proposal=False): diff --git a/pymisp/data/describeTypes.json b/pymisp/data/describeTypes.json new file mode 100644 index 0000000..5ac2edd --- /dev/null +++ b/pymisp/data/describeTypes.json @@ -0,0 +1 @@ +{"result":{"sane_defaults":{"md5":{"default_category":"Payload delivery","to_ids":1},"sha1":{"default_category":"Payload delivery","to_ids":1},"sha256":{"default_category":"Payload delivery","to_ids":1},"filename":{"default_category":"Payload delivery","to_ids":1},"pdb":{"default_category":"Artifacts dropped","to_ids":0},"filename|md5":{"default_category":"Payload delivery","to_ids":1},"filename|sha1":{"default_category":"Payload delivery","to_ids":1},"filename|sha256":{"default_category":"Payload delivery","to_ids":1},"ip-src":{"default_category":"Network activity","to_ids":1},"ip-dst":{"default_category":"Network activity","to_ids":1},"hostname":{"default_category":"Network activity","to_ids":1},"domain":{"default_category":"Network activity","to_ids":1},"domain|ip":{"default_category":"Network activity","to_ids":1},"email-src":{"default_category":"Payload delivery","to_ids":1},"email-dst":{"default_category":"Network activity","to_ids":1},"email-subject":{"default_category":"Payload delivery","to_ids":0},"email-attachment":{"default_category":"Payload delivery","to_ids":1},"url":{"default_category":"External analysis","to_ids":1},"http-method":{"default_category":"Network activity","to_ids":0},"user-agent":{"default_category":"Network activity","to_ids":0},"regkey":{"default_category":"Persistence mechanism","to_ids":1},"regkey|value":{"default_category":"Persistence mechanism","to_ids":1},"AS":{"default_category":"Network activity","to_ids":0},"snort":{"default_category":"Network activity","to_ids":1},"pattern-in-file":{"default_category":"Payload installation","to_ids":1},"pattern-in-traffic":{"default_category":"Network activity","to_ids":1},"pattern-in-memory":{"default_category":"Payload installation","to_ids":1},"yara":{"default_category":"Payload installation","to_ids":1},"vulnerability":{"default_category":"External analysis","to_ids":0},"attachment":{"default_category":"External analysis","to_ids":0},"malware-sample":{"default_category":"Payload delivery","to_ids":1},"link":{"default_category":"External analysis","to_ids":0},"comment":{"default_category":"Other","to_ids":0},"text":{"default_category":"Other","to_ids":0},"other":{"default_category":"Other","to_ids":0},"named pipe":{"default_category":"Artifacts dropped","to_ids":0},"mutex":{"default_category":"Artifacts dropped","to_ids":1},"target-user":{"default_category":"Targeting data","to_ids":0},"target-email":{"default_category":"Targeting data","to_ids":0},"target-machine":{"default_category":"Targeting data","to_ids":0},"target-org":{"default_category":"Targeting data","to_ids":0},"target-location":{"default_category":"Targeting data","to_ids":0},"target-external":{"default_category":"Targeting data","to_ids":0},"btc":{"default_category":"Financial fraud","to_ids":1},"iban":{"default_category":"Financial fraud","to_ids":1},"bic":{"default_category":"Financial fraud","to_ids":1},"bank-account-nr":{"default_category":"Financial fraud","to_ids":1},"aba-rtn":{"default_category":"Financial fraud","to_ids":1},"bin":{"default_category":"Financial fraud","to_ids":1},"cc-number":{"default_category":"Financial fraud","to_ids":1},"prtn":{"default_category":"Financial fraud","to_ids":1},"threat-actor":{"default_category":"Attribution","to_ids":0},"campaign-name":{"default_category":"Attribution","to_ids":0},"campaign-id":{"default_category":"Attribution","to_ids":0},"malware-type":{"default_category":"Payload delivery","to_ids":0},"uri":{"default_category":"Network activity","to_ids":1},"authentihash":{"default_category":"Payload delivery","to_ids":1},"ssdeep":{"default_category":"Payload delivery","to_ids":1},"imphash":{"default_category":"Payload delivery","to_ids":1},"pehash":{"default_category":"Payload delivery","to_ids":1},"sha224":{"default_category":"Payload delivery","to_ids":1},"sha384":{"default_category":"Payload delivery","to_ids":1},"sha512":{"default_category":"Payload delivery","to_ids":1},"sha512\/224":{"default_category":"Payload delivery","to_ids":1},"sha512\/256":{"default_category":"Payload delivery","to_ids":1},"tlsh":{"default_category":"Payload delivery","to_ids":1},"filename|authentihash":{"default_category":"Payload delivery","to_ids":1},"filename|ssdeep":{"default_category":"Payload delivery","to_ids":1},"filename|imphash":{"default_category":"Payload delivery","to_ids":1},"filename|pehash":{"default_category":"Payload delivery","to_ids":1},"filename|sha224":{"default_category":"Payload delivery","to_ids":1},"filename|sha384":{"default_category":"Payload delivery","to_ids":1},"filename|sha512":{"default_category":"Payload delivery","to_ids":1},"filename|sha512\/224":{"default_category":"Payload delivery","to_ids":1},"filename|sha512\/256":{"default_category":"Payload delivery","to_ids":1},"filename|tlsh":{"default_category":"Payload delivery","to_ids":1},"windows-scheduled-task":{"default_category":"Artifacts dropped","to_ids":0},"windows-service-name":{"default_category":"Artifacts dropped","to_ids":0},"windows-service-displayname":{"default_category":"Artifacts dropped","to_ids":0},"whois-registrant-email":{"default_category":"Attribution","to_ids":0},"whois-registrant-phone":{"default_category":"Attribution","to_ids":0},"whois-registrant-name":{"default_category":"Attribution","to_ids":0},"whois-registrar":{"default_category":"Attribution","to_ids":0},"whois-creation-date":{"default_category":"Attribution","to_ids":0},"x509-fingerprint-sha1":{"default_category":"Network activity","to_ids":1}},"types":["md5","sha1","sha256","filename","pdb","filename|md5","filename|sha1","filename|sha256","ip-src","ip-dst","hostname","domain","domain|ip","email-src","email-dst","email-subject","email-attachment","url","http-method","user-agent","regkey","regkey|value","AS","snort","pattern-in-file","pattern-in-traffic","pattern-in-memory","yara","vulnerability","attachment","malware-sample","link","comment","text","other","named pipe","mutex","target-user","target-email","target-machine","target-org","target-location","target-external","btc","iban","bic","bank-account-nr","aba-rtn","bin","cc-number","prtn","threat-actor","campaign-name","campaign-id","malware-type","uri","authentihash","ssdeep","imphash","pehash","sha224","sha384","sha512","sha512\/224","sha512\/256","tlsh","filename|authentihash","filename|ssdeep","filename|imphash","filename|pehash","filename|sha224","filename|sha384","filename|sha512","filename|sha512\/224","filename|sha512\/256","filename|tlsh","windows-scheduled-task","windows-service-name","windows-service-displayname","whois-registrant-email","whois-registrant-phone","whois-registrant-name","whois-registrar","whois-creation-date","x509-fingerprint-sha1"],"categories":["Internal reference","Targeting data","Antivirus detection","Payload delivery","Artifacts dropped","Payload installation","Persistence mechanism","Network activity","Payload type","Attribution","External analysis","Financial fraud","Other"],"category_type_mappings":{"Internal reference":["text","link","comment","other"],"Targeting data":["target-user","target-email","target-machine","target-org","target-location","target-external","comment"],"Antivirus detection":["link","comment","text","attachment","other"],"Payload delivery":["md5","sha1","sha224","sha256","sha384","sha512","sha512\/224","sha512\/256","ssdeep","imphash","authentihash","pehash","tlsh","filename","filename|md5","filename|sha1","filename|sha224","filename|sha256","filename|sha384","filename|sha512","filename|sha512\/224","filename|sha512\/256","filename|authentihash","filename|ssdeep","filename|tlsh","filename|imphash","filename|pehash","ip-src","ip-dst","hostname","domain","email-src","email-dst","email-subject","email-attachment","url","user-agent","AS","pattern-in-file","pattern-in-traffic","yara","attachment","malware-sample","link","malware-type","comment","text","vulnerability","x509-fingerprint-sha1","other"],"Artifacts dropped":["md5","sha1","sha224","sha256","sha384","sha512","sha512\/224","sha512\/256","ssdeep","imphash","authentihash","filename","filename|md5","filename|sha1","filename|sha224","filename|sha256","filename|sha384","filename|sha512","filename|sha512\/224","filename|sha512\/256","filename|authentihash","filename|ssdeep","filename|tlsh","filename|imphash","filename|pehash","regkey","regkey|value","pattern-in-file","pattern-in-memory","pdb","yara","attachment","malware-sample","named pipe","mutex","windows-scheduled-task","windows-service-name","windows-service-displayname","comment","text","x509-fingerprint-sha1","other"],"Payload installation":["md5","sha1","sha224","sha256","sha384","sha512","sha512\/224","sha512\/256","ssdeep","imphash","authentihash","pehash","tlsh","filename","filename|md5","filename|sha1","filename|sha224","filename|sha256","filename|sha384","filename|sha512","filename|sha512\/224","filename|sha512\/256","filename|authentihash","filename|ssdeep","filename|tlsh","filename|imphash","filename|pehash","pattern-in-file","pattern-in-traffic","pattern-in-memory","yara","vulnerability","attachment","malware-sample","malware-type","comment","text","x509-fingerprint-sha1","other"],"Persistence mechanism":["filename","regkey","regkey|value","comment","text","other"],"Network activity":["ip-src","ip-dst","hostname","domain","domain|ip","email-dst","url","uri","user-agent","http-method","AS","snort","pattern-in-file","pattern-in-traffic","attachment","comment","text","x509-fingerprint-sha1","other"],"Payload type":["comment","text","other"],"Attribution":["threat-actor","campaign-name","campaign-id","whois-registrant-phone","whois-registrant-email","whois-registrant-name","whois-registrar","whois-creation-date","comment","text","x509-fingerprint-sha1","other"],"External analysis":["md5","sha1","sha256","filename","filename|md5","filename|sha1","filename|sha256","ip-src","ip-dst","hostname","domain","domain|ip","url","user-agent","regkey","regkey|value","AS","snort","pattern-in-file","pattern-in-traffic","pattern-in-memory","vulnerability","attachment","malware-sample","link","comment","text","x509-fingerprint-sha1","other"],"Financial fraud":["btc","iban","bic","bank-account-nr","aba-rtn","bin","cc-number","prtn","comment","text","other"],"Other":["comment","text","other"]}}} \ No newline at end of file diff --git a/pymisp/data/schema-lax.json b/pymisp/data/schema-lax.json new file mode 100644 index 0000000..09bc780 --- /dev/null +++ b/pymisp/data/schema-lax.json @@ -0,0 +1,322 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json", + "type": "object", + "properties": { + "Event": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event", + "type": "object", + "properties": { + "id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/id", + "type": "string" + }, + "orgc_id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/orgc_id", + "type": "string" + }, + "org_id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/org_id", + "type": "string" + }, + "date": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/date", + "type": "string" + }, + "threat_level_id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/threat_level_id", + "type": "string" + }, + "info": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/info", + "type": "string" + }, + "published": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/published", + "type": "boolean" + }, + "uuid": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/uuid", + "type": "string" + }, + "attribute_count": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/attribute_count", + "type": "string" + }, + "analysis": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/analysis", + "type": "string" + }, + "timestamp": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/timestamp", + "type": "string" + }, + "distribution": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/distribution", + "type": "string" + }, + "proposal_email_lock": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/proposal_email_lock", + "type": "boolean" + }, + "locked": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/locked", + "type": "boolean" + }, + "publish_timestamp": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/publish_timestamp", + "type": "string" + }, + "sharing_group_id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/sharing_group_id", + "type": "string" + }, + "Org": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Org", + "type": "object", + "properties": { + "id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Org/id", + "type": "string" + }, + "name": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Org/name", + "type": "string" + }, + "uuid": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Org/uuid", + "type": "string" + } + } + }, + "Orgc": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Orgc", + "type": "object", + "properties": { + "id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Orgc/id", + "type": "string" + }, + "name": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Orgc/name", + "type": "string" + }, + "uuid": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Orgc/uuid", + "type": "string" + } + } + }, + "Attribute": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute", + "type": "array", + "items": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17", + "type": "object", + "properties": { + "id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17/id", + "type": "string" + }, + "type": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17/type", + "type": "string" + }, + "category": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17/category", + "type": "string" + }, + "to_ids": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17/to_ids", + "type": "boolean" + }, + "uuid": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17/uuid", + "type": "string" + }, + "event_id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17/event_id", + "type": "string" + }, + "distribution": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17/distribution", + "type": "string" + }, + "timestamp": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17/timestamp", + "type": "string" + }, + "comment": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17/comment", + "type": "string" + }, + "sharing_group_id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17/sharing_group_id", + "type": "string" + }, + "value": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17/value", + "type": "string" + }, + "SharingGroup": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17/SharingGroup", + "type": "array", + "items": {}, + "additionalItems": false + }, + "ShadowAttribute": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17/ShadowAttribute", + "type": "array", + "items": {}, + "additionalItems": false + } + } + }, + "additionalItems": false + }, + "ShadowAttribute": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/ShadowAttribute", + "type": "array", + "items": {}, + "additionalItems": false + }, + "RelatedEvent": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent", + "type": "array", + "items": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0", + "type": "object", + "properties": { + "Org": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Org", + "type": "object", + "properties": { + "id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Org/id", + "type": "string" + }, + "name": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Org/name", + "type": "string" + }, + "uuid": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Org/uuid", + "type": "string" + } + } + }, + "Orgc": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Orgc", + "type": "object", + "properties": { + "id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Orgc/id", + "type": "string" + }, + "name": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Orgc/name", + "type": "string" + }, + "uuid": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Orgc/uuid", + "type": "string" + } + } + }, + "Event": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event", + "type": "object", + "items": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0", + "type": "object", + "properties": { + "id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0/id", + "type": "string" + }, + "date": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0/date", + "type": "string" + }, + "threat_level_id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0/threat_level_id", + "type": "string" + }, + "info": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0/info", + "type": "string" + }, + "published": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0/published", + "type": "boolean" + }, + "uuid": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0/uuid", + "type": "string" + }, + "analysis": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0/analysis", + "type": "string" + }, + "timestamp": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0/timestamp", + "type": "string" + }, + "distribution": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0/distribution", + "type": "string" + }, + "org_id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0/org_id", + "type": "string" + }, + "orgc_id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0/orgc_id", + "type": "string" + } + } + }, + "additionalItems": false + } + } + }, + "additionalItems": false + }, + "Tag": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Tag", + "type": "array", + "items": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Tag/2", + "type": "object", + "properties": { + "id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Tag/2/id", + "type": "string" + }, + "name": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Tag/2/name", + "type": "string" + }, + "colour": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Tag/2/colour", + "type": "string" + }, + "exportable": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Tag/2/exportable", + "type": "boolean" + } + } + }, + "additionalItems": false + } + }, + "required": [ + "info", + "Attribute" + ] + } + }, + "required": [ + "Event" + ] +} diff --git a/pymisp/data/schema.json b/pymisp/data/schema.json new file mode 100644 index 0000000..4560498 --- /dev/null +++ b/pymisp/data/schema.json @@ -0,0 +1,327 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json", + "type": "object", + "properties": { + "Event": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event", + "type": "object", + "properties": { + "id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/id", + "type": "integer" + }, + "orgc_id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/orgc_id", + "type": "integer" + }, + "org_id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/org_id", + "type": "integer" + }, + "date": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/date", + "type": "string" + }, + "threat_level_id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/threat_level_id", + "type": "integer" + }, + "info": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/info", + "type": "string" + }, + "published": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/published", + "type": "boolean" + }, + "uuid": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/uuid", + "type": "string" + }, + "attribute_count": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/attribute_count", + "type": "integer" + }, + "analysis": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/analysis", + "type": "integer" + }, + "timestamp": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/timestamp", + "type": "integer" + }, + "distribution": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/distribution", + "type": "integer" + }, + "proposal_email_lock": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/proposal_email_lock", + "type": "boolean" + }, + "locked": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/locked", + "type": "boolean" + }, + "publish_timestamp": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/publish_timestamp", + "type": "integer" + }, + "sharing_group_id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/sharing_group_id", + "type": "integer" + }, + "Org": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Org", + "type": "object", + "properties": { + "id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Org/id", + "type": "string" + }, + "name": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Org/name", + "type": "string" + }, + "uuid": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Org/uuid", + "type": "string" + } + } + }, + "Orgc": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Orgc", + "type": "object", + "properties": { + "id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Orgc/id", + "type": "string" + }, + "name": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Orgc/name", + "type": "string" + }, + "uuid": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Orgc/uuid", + "type": "string" + } + } + }, + "Attribute": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute", + "type": "array", + "items": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17", + "type": "object", + "properties": { + "id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17/id", + "type": "string" + }, + "type": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17/type", + "type": "string" + }, + "category": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17/category", + "type": "string" + }, + "to_ids": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17/to_ids", + "type": "boolean" + }, + "uuid": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17/uuid", + "type": "string" + }, + "event_id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17/event_id", + "type": "integer" + }, + "distribution": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17/distribution", + "type": "integer" + }, + "timestamp": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17/timestamp", + "type": "integer" + }, + "comment": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17/comment", + "type": "string" + }, + "sharing_group_id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17/sharing_group_id", + "type": "integer" + }, + "value": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17/value", + "type": "string" + }, + "SharingGroup": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17/SharingGroup", + "type": "array", + "items": {}, + "additionalItems": false + }, + "ShadowAttribute": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17/ShadowAttribute", + "type": "array", + "items": {}, + "additionalItems": false + } + } + }, + "additionalItems": false + }, + "ShadowAttribute": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/ShadowAttribute", + "type": "array", + "items": {}, + "additionalItems": false + }, + "RelatedEvent": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent", + "type": "array", + "items": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0", + "type": "object", + "properties": { + "Org": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Org", + "type": "object", + "properties": { + "id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Org/id", + "type": "string" + }, + "name": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Org/name", + "type": "string" + }, + "uuid": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Org/uuid", + "type": "string" + } + } + }, + "Orgc": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Orgc", + "type": "object", + "properties": { + "id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Orgc/id", + "type": "string" + }, + "name": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Orgc/name", + "type": "string" + }, + "uuid": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Orgc/uuid", + "type": "string" + } + } + }, + "Event": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event", + "type": "object", + "items": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0", + "type": "object", + "properties": { + "id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0/id", + "type": "string" + }, + "date": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0/date", + "type": "string" + }, + "threat_level_id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0/threat_level_id", + "type": "integer" + }, + "info": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0/info", + "type": "string" + }, + "published": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0/published", + "type": "boolean" + }, + "uuid": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0/uuid", + "type": "string" + }, + "analysis": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0/analysis", + "type": "string" + }, + "timestamp": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0/timestamp", + "type": "integer" + }, + "distribution": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0/distribution", + "type": "string" + }, + "org_id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0/org_id", + "type": "integer" + }, + "orgc_id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0/orgc_id", + "type": "integer" + } + } + }, + "additionalItems": false + } + } + }, + "additionalItems": false + }, + "Tag": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Tag", + "type": "array", + "items": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Tag/2", + "type": "object", + "properties": { + "id": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Tag/2/id", + "type": "string" + }, + "name": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Tag/2/name", + "type": "string" + }, + "colour": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Tag/2/colour", + "type": "string" + }, + "exportable": { + "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Tag/2/exportable", + "type": "boolean" + } + } + }, + "additionalItems": false + } + }, + "required": [ + "date", + "threat_level_id", + "info", + "published", + "analysis", + "distribution", + "Attribute" + ] + } + }, + "required": [ + "Event" + ] +} diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 2d181b4..4e9d2b6 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -4,6 +4,17 @@ import datetime import time import json +from json import JSONEncoder +import os +try: + from dateutil.parser import parse +except ImportError: + pass + +try: + import jsonschema +except ImportError: + pass from .exceptions import PyMISPError, NewEventError, NewAttributeError @@ -14,8 +25,9 @@ class MISPAttribute(object): self.categories = categories self.types = types self.category_type_mapping = category_type_mapping - self.new = True + self._reinitialize_attribute() + def _reinitialize_attribute(self): # Default values self.category = None self.type = None @@ -24,53 +36,106 @@ class MISPAttribute(object): self.comment = '' self.distribution = 5 - def set_values(self, type_value, value, category, to_ids, comment, distribution): - self._validate(type_value, value, category, to_ids, comment, distribution) - self.type = type_value - self.value = value - self.category = category - self.to_ids = to_ids - self.comment = comment - self.distribution = distribution + # other possible values + self.id = None + self.uuid = None + self.timestamp = None + self.sharing_group_id = None + self.deleted = None + self.SharingGroup = [] + self.ShadowAttribute = [] - def set_values_existing_attribute(self, attribute_id, uuid, timestamp, sharing_group_id, deleted, SharingGroup, ShadowAttribute): - self.new = False - self.id = int(attribute_id) - self.uuid = uuid - self.timestamp = datetime.datetime.fromtimestamp(timestamp) - self.sharing_group_id = int(sharing_group_id) - self.deleted = deleted - self.SharingGroup = SharingGroup - self.ShadowAttribute = ShadowAttribute + def set_all_values(self, **kwargs): + # Default values + if kwargs.get('category', None): + self.category = kwargs['category'] + if self.category not in self.categories: + raise NewAttributeError('{} is invalid, category has to be in {}'.format(self.category, (', '.join(self.categories)))) + if kwargs.get('type', None): + self.type = kwargs['type'] + if self.type not in self.types: + raise NewAttributeError('{} is invalid, type_value has to be in {}'.format(self.type, (', '.join(self.types)))) + if self.type not in self.category_type_mapping[self.category]: + raise NewAttributeError('{} and {} is an invalid combinaison, type_value for this category has to be in {}'.capitalizeformat(self.type, self.category, (', '.join(self.category_type_mapping[self.category])))) + if kwargs.get('value', None): + self.value = kwargs['value'] + if kwargs.get('to_ids', None): + self.to_ids = kwargs['to_ids'] + if not isinstance(self.to_ids, bool): + raise NewAttributeError('{} is invalid, to_ids has to be True or False'.format(self.to_ids)) + if kwargs.get('comment', None): + self.comment = kwargs['comment'] + if kwargs.get('distribution', None): + self.distribution = int(kwargs['distribution']) + if self.distribution not in [0, 1, 2, 3, 5]: + raise NewAttributeError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 5'.format(self.distribution)) - def _validate(self, type_value, value, category, to_ids, comment, distribution): - if category not in self.categories: - raise NewAttributeError('{} is invalid, category has to be in {}'.format(category, (', '.join(self.categories)))) - if type_value not in self.types: - raise NewAttributeError('{} is invalid, type_value has to be in {}'.format(type_value, (', '.join(self.types)))) - if type_value not in self.category_type_mapping[category]: - raise NewAttributeError('{} and {} is an invalid combinaison, type_value for this category has to be in {}'.capitalizeformat(type_value, category, (', '.join(self.category_type_mapping[category])))) - if to_ids not in [True, False]: - raise NewAttributeError('{} is invalid, to_ids has to be True or False'.format(to_ids)) - if distribution not in [0, 1, 2, 3, 5]: - raise NewAttributeError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 5'.format(distribution)) + # other possible values + if kwargs.get('id', None): + self.id = int(kwargs['id']) + if kwargs.get('uuid', None): + self.uuid = kwargs['uuid'] + if kwargs.get('timestamp', None): + self.timestamp = datetime.datetime.fromtimestamp(int(kwargs['timestamp'])) + if kwargs.get('sharing_group_id', None): + self.sharing_group_id = int(kwargs['sharing_group_id']) + if kwargs.get('deleted', None): + self.deleted = kwargs['deleted'] + if kwargs.get('SharingGroup', None): + self.SharingGroup = kwargs['SharingGroup'] + if kwargs.get('ShadowAttribute', None): + self.ShadowAttribute = kwargs['ShadowAttribute'] - def dump(self): + def _json(self): to_return = {'type': self.type, 'category': self.category, 'to_ids': self.to_ids, 'distribution': self.distribution, 'value': self.value, 'comment': self.comment} - if not self.new: - to_return.update( - {'id': self.id, 'uuid': self.uuid, - 'timestamp': int(time.mktime(self.timestamp.timetuple())), - 'sharing_group_id': self.sharing_group_id, 'deleted': self.deleted, - 'SharingGroup': self.SharingGroup, 'ShadowAttribute': self.ShadowAttribute}) + if self.sharing_group_id: + to_return['sharing_group_id'] = self.sharing_group_id return to_return + def _json_full(self): + to_return = self._json() + if self.id: + to_return['id'] = self.id + if self.uuid: + to_return['uuid'] = self.uuid + if self.timestamp: + to_return['timestamp'] = int(time.mktime(self.timestamp.timetuple())) + if self.deleted is not None: + to_return['deleted'] = self.deleted + if self.ShadowAttribute: + to_return['ShadowAttribute'] = self.ShadowAttribute + if self.SharingGroup: + to_return['SharingGroup'] = self.SharingGroup + return to_return + + +class EncodeUpdate(JSONEncoder): + def default(self, obj): + try: + return obj._json() + except AttributeError: + return JSONEncoder.default(self, obj) + + +class EncodeFull(JSONEncoder): + def default(self, obj): + try: + return obj._json_full() + except AttributeError: + return JSONEncoder.default(self, obj) + class MISPEvent(object): - def __init__(self, describe_types): + def __init__(self, describe_types=None): + self.ressources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') + self.json_schema = json.load(open(os.path.join(self.ressources_path, 'schema.json'), 'r')) + self.json_schema_lax = json.load(open(os.path.join(self.ressources_path, 'schema-lax.json'), 'r')) + if not describe_types: + t = json.load(open(os.path.join(self.ressources_path, 'describeTypes.json'), 'r')) + describe_types = t['result'] self.categories = describe_types['categories'] self.types = describe_types['types'] self.category_type_mapping = describe_types['category_type_mappings'] @@ -78,7 +143,10 @@ class MISPEvent(object): self.new = True self.dump_full = False - # Default values + self._reinitialize_event() + + def _reinitialize_event(self): + # Default values for a valid event to send to a MISP instance self.distribution = 3 self.threat_level_id = 2 self.analysis = 0 @@ -87,104 +155,159 @@ class MISPEvent(object): self.date = datetime.date.today() self.attributes = [] - def _validate(self, distribution, threat_level_id, analysis): - if distribution not in [0, 1, 2, 3]: - raise NewEventError('{} is invalid, the distribution has to be in 0, 1, 2, 3'.format(distribution)) - if threat_level_id not in [1, 2, 3, 4]: - raise NewEventError('{} is invalid, the threat_level has to be in 1, 2, 3, 4'.format(threat_level_id)) - if analysis not in [0, 1, 2]: - raise NewEventError('{} is invalid, the analysis has to be in 0, 1, 2'.format(analysis)) + # All other keys + self.id = None + self.orgc_id = None + self.org_id = None + self.uuid = None + self.attribute_count = None + self.timestamp = None + self.proposal_email_lock = None + self.locked = None + self.publish_timestamp = None + self.sharing_group_id = None + self.Org = None + self.Orgc = None + self.ShadowAttribute = [] + self.RelatedEvent = [] + self.Tag = [] def load(self, json_event): self.new = False self.dump_full = True - loaded = json.loads(json_event) - if loaded.get('response'): - e = loaded.get('response')[0].get('Event') + if isinstance(json_event, str): + loaded = json.loads(json_event) + if loaded.get('response'): + event = loaded.get('response')[0] + else: + event = loaded + if not event: + raise PyMISPError('Invalid event') else: - e = loaded.get('Event') - if not e: - raise PyMISPError('Invalid event') - try: - date = datetime.date(*map(int, e['date'].split('-'))) - except: - raise NewEventError('{} is an invalid date.'.format(e['date'])) - self.set_values(e['info'], int(e['distribution']), int(e['threat_level_id']), int(e['analysis']), date) - if e['published']: - self.publish() - self.set_values_existing_event( - e['id'], e['orgc_id'], e['org_id'], e['uuid'], - e['attribute_count'], e['proposal_email_lock'], e['locked'], - e['publish_timestamp'], e['sharing_group_id'], e['Org'], e['Orgc'], - e['ShadowAttribute'], e['RelatedEvent']) - self.attributes = [] - for a in e['Attribute']: - attribute = MISPAttribute(self.categories, self.types, self.category_type_mapping) - attribute.set_values(a['type'], a['value'], a['category'], a['to_ids'], - a['comment'], int(a['distribution'])) - attribute.set_values_existing_attribute(a['id'], a['uuid'], a['timestamp'], - a['sharing_group_id'], a['deleted'], - a['SharingGroup'], a['ShadowAttribute']) - self.attributes.append(attribute) + event = json_event + jsonschema.validate(event, self.json_schema_lax) + e = event.get('Event') + self._reinitialize_event() + self.set_all_values(**e) - def dump(self): + def set_all_values(self, **kwargs): + # Default values for a valid event to send to a MISP instance + if kwargs.get('distribution', None) is not None: + self.distribution = int(kwargs['distribution']) + if self.distribution not in [0, 1, 2, 3]: + raise NewEventError('{} is invalid, the distribution has to be in 0, 1, 2, 3'.format(self.distribution)) + if kwargs.get('threat_level_id', None) is not None: + self.threat_level_id = int(kwargs['threat_level_id']) + if self.threat_level_id not in [1, 2, 3, 4]: + raise NewEventError('{} is invalid, the threat_level has to be in 1, 2, 3, 4'.format(self.threat_level_id)) + if kwargs.get('analysis', None) is not None: + self.analysis = int(kwargs['analysis']) + if self.analysis not in [0, 1, 2]: + raise NewEventError('{} is invalid, the analysis has to be in 0, 1, 2'.format(self.analysis)) + if kwargs.get('info', None): + self.info = kwargs['info'] + if kwargs.get('published', None) is not None: + self.publish() + if kwargs.get('date', None): + if isinstance(kwargs['date'], str): + self.date = parse(kwargs['date']) + elif isinstance(kwargs['date'], datetime.datetime): + self.date = kwargs['date'].date() + elif isinstance(kwargs['date'], datetime.date): + self.date = kwargs['date'] + else: + raise NewEventError('Invalid format for the date: {} - {}'.format(kwargs['date'], type(kwargs['date']))) + if kwargs.get('Attribute', None): + for a in kwargs['Attribute']: + attribute = MISPAttribute(self.categories, self.types, self.category_type_mapping) + attribute.set_all_values(**a) + self.attributes.append(attribute) + + # All other keys + if kwargs.get('id', None): + self.id = int(kwargs['id']) + if kwargs.get('orgc_id', None): + self.orgc_id = int(kwargs['orgc_id']) + if kwargs.get('org_id', None): + self.org_id = int(kwargs['org_id']) + if kwargs.get('uuid', None): + self.uuid = kwargs['uuid'] + if kwargs.get('attribute_count', None): + self.attribute_count = int(kwargs['attribute_count']) + if kwargs.get('timestamp', None): + self.timestamp = datetime.datetime.fromtimestamp(int(kwargs['timestamp'])) + if kwargs.get('proposal_email_lock', None): + self.proposal_email_lock = kwargs['proposal_email_lock'] + if kwargs.get('locked', None): + self.locked = kwargs['locked'] + if kwargs.get('publish_timestamp', None): + self.publish_timestamp = datetime.datetime.fromtimestamp(int(kwargs['publish_timestamp'])) + if kwargs.get('sharing_group_id', None): + self.sharing_group_id = int(kwargs['sharing_group_id']) + if kwargs.get('Org', None): + self.Org = kwargs['Org'] + if kwargs.get('Orgc', None): + self.Orgc = kwargs['Orgc'] + if kwargs.get('ShadowAttribute', None): + self.ShadowAttribute = kwargs['ShadowAttribute'] + if kwargs.get('RelatedEvent', None): + self.RelatedEvent = kwargs['RelatedEvent'] + if kwargs.get('Tag', None): + self.Tag = kwargs['Tag'] + + def _json(self): to_return = {'Event': {}} to_return['Event'] = {'distribution': self.distribution, 'info': self.info, 'date': self.date.isoformat(), 'published': self.published, 'threat_level_id': self.threat_level_id, 'analysis': self.analysis, 'Attribute': []} - if not self.new: - to_return['Event'].update( - {'id': self.id, 'orgc_id': self.orgc_id, 'org_id': self.org_id, - 'uuid': self.uuid, 'sharing_group_id': self.sharing_group_id}) - if self.dump_full: - to_return['Event'].update( - {'locked': self.locked, 'attribute_count': self.attribute_count, - 'RelatedEvent': self.RelatedEvent, 'Orgc': self.Orgc, - 'ShadowAttribute': self.ShadowAttribute, 'Org': self.Org, - 'proposal_email_lock': self.proposal_email_lock, - 'publish_timestamp': int(time.mktime(self.publish_timestamp.timetuple()))}) - to_return['Event']['Attribute'] = [a.dump() for a in self.attributes] - return json.dumps(to_return) + if self.id: + to_return['Event']['id'] = self.id + if self.orgc_id: + to_return['Event']['orgc_id'] = self.orgc_id + if self.org_id: + to_return['Event']['org_id'] = self.org_id + if self.uuid: + to_return['Event']['uuid'] = self.uuid + if self.sharing_group_id: + to_return['Event']['sharing_group_id'] = self.sharing_group_id + if self.Tag: + to_return['Event']['Tag'] = self.Tag + to_return['Event']['Attribute'] = [a._json() for a in self.attributes] + jsonschema.validate(to_return, self.json_schema) + return to_return - def set_values(self, info, distribution=3, threat_level_id=2, analysis=0, date=None): - self._validate(distribution, threat_level_id, analysis) - self.info = info - self.distribution = distribution - self.threat_level_id = threat_level_id - self.analysis = analysis - if not date: - self.date = datetime.date.today() - else: - self.date = date - - def set_values_existing_event(self, event_id, orgc_id, org_id, uuid, attribute_count, - proposal_email_lock, locked, publish_timestamp, - sharing_group_id, Org, Orgc, ShadowAttribute, - RelatedEvent): - self.id = int(event_id) - self.orgc_id = int(orgc_id) - self.org_id = int(org_id) - self.uuid = uuid - self.attribute_count = int(attribute_count) - self.proposal_email_lock = proposal_email_lock - self.locked = locked - self.publish_timestamp = datetime.datetime.fromtimestamp(publish_timestamp) - self.sharing_group_id = int(sharing_group_id) - self.Org = Org - self.Orgc = Orgc - self.ShadowAttribute = ShadowAttribute - self.RelatedEvent = RelatedEvent + def _json_full(self): + to_return = self._json() + if self.locked is not None: + to_return['Event']['locked'] = self.locked + if self.attribute_count: + to_return['Event']['attribute_count'] = self.attribute_count + if self.RelatedEvent: + to_return['Event']['RelatedEvent'] = self.RelatedEvent + if self.Org: + to_return['Event']['Org'] = self.Org + if self.Orgc: + to_return['Event']['Orgc'] = self.Orgc + if self.ShadowAttribute: + to_return['Event']['ShadowAttribute'] = self.ShadowAttribute + if self.proposal_email_lock is not None: + to_return['Event']['proposal_email_lock'] = self.proposal_email_lock + if self.locked is not None: + to_return['Event']['locked'] = self.locked + if self.publish_timestamp: + to_return['Event']['publish_timestamp'] = int(time.mktime(self.publish_timestamp.timetuple())) + if self.timestamp: + to_return['Event']['timestamp'] = int(time.mktime(self.timestamp.timetuple())) + to_return['Event']['Attribute'] = [a._json_full() for a in self.attributes] + jsonschema.validate(to_return, self.json_schema) + return to_return def publish(self): - self.publish = True + self.published = True def unpublish(self): - self.publish = False - - def prepare_for_update(self): - self.unpublish() - self.dump_full = False + self.published = False def add_attribute(self, type_value, value, **kwargs): if not self.sane_default.get(type_value): @@ -207,5 +330,6 @@ class MISPEvent(object): else: distribution = 5 attribute = MISPAttribute(self.categories, self.types, self.category_type_mapping) - attribute.set_values(type_value, value, category, to_ids, comment, distribution) + attribute.set_all_values(type=type_value, value=value, category=category, + to_ids=to_ids, comment=comment, distribution=distribution) self.attributes.append(attribute) diff --git a/setup.py b/setup.py index 5cab2ca..7a465ac 100644 --- a/setup.py +++ b/setup.py @@ -27,5 +27,7 @@ setup( 'Topic :: Internet', ], test_suite="tests", - install_requires=['requests'], + install_requires=['requests', 'python-dateutil', 'jsonschema'], + include_package_data=True, + package_data={'data': ['schema.json', 'schema-lax.json', 'describeTypes.json']}, ) diff --git a/tests/describeTypes.json b/tests/describeTypes.json deleted file mode 100644 index bac9389..0000000 --- a/tests/describeTypes.json +++ /dev/null @@ -1,368 +0,0 @@ -{ - "result": { - "types": [ - "md5", - "sha1", - "sha256", - "filename", - "pdb", - "filename|md5", - "filename|sha1", - "filename|sha256", - "ip-src", - "ip-dst", - "hostname", - "domain", - "domain|ip", - "email-src", - "email-dst", - "email-subject", - "email-attachment", - "url", - "http-method", - "user-agent", - "regkey", - "regkey|value", - "AS", - "snort", - "pattern-in-file", - "pattern-in-traffic", - "pattern-in-memory", - "yara", - "vulnerability", - "attachment", - "malware-sample", - "link", - "comment", - "text", - "other", - "named pipe", - "mutex", - "target-user", - "target-email", - "target-machine", - "target-org", - "target-location", - "target-external", - "btc", - "iban", - "bic", - "bank-account-nr", - "aba-rtn", - "bin", - "cc-number", - "prtn", - "threat-actor", - "campaign-name", - "campaign-id", - "malware-type", - "uri", - "authentihash", - "ssdeep", - "imphash", - "pehash", - "sha224", - "sha384", - "sha512", - "sha512/224", - "sha512/256", - "tlsh", - "filename|authentihash", - "filename|ssdeep", - "filename|imphash", - "filename|pehash", - "filename|sha224", - "filename|sha384", - "filename|sha512", - "filename|sha512/224", - "filename|sha512/256", - "filename|tlsh", - "windows-scheduled-task", - "windows-service-name", - "windows-service-displayname", - "whois-registrant-email", - "whois-registrant-phone", - "whois-registrant-name", - "whois-registrar", - "whois-creation-date", - "targeted-threat-index", - "mailslot", - "pipe", - "ssl-cert-attributes", - "x509-fingerprint-sha1" - ], - "categories": [ - "Internal reference", - "Targeting data", - "Antivirus detection", - "Payload delivery", - "Artifacts dropped", - "Payload installation", - "Persistence mechanism", - "Network activity", - "Payload type", - "Attribution", - "External analysis", - "Financial fraud", - "Other" - ], - "category_type_mappings": { - "Internal reference": [ - "link", - "comment", - "text", - "other" - ], - "Targeting data": [ - "target-user", - "target-email", - "target-machine", - "target-org", - "target-location", - "target-external", - "comment" - ], - "Antivirus detection": [ - "link", - "comment", - "text", - "attachment", - "other" - ], - "Payload delivery": [ - "md5", - "sha1", - "sha224", - "sha256", - "sha384", - "sha512", - "sha512/224", - "sha512/256", - "ssdeep", - "imphash", - "authentihash", - "pehash", - "tlsh", - "filename", - "filename|md5", - "filename|sha1", - "filename|sha224", - "filename|sha256", - "filename|sha384", - "filename|sha512", - "filename|sha512/224", - "filename|sha512/256", - "filename|authentihash", - "filename|ssdeep", - "filename|tlsh", - "filename|imphash", - "filename|pehash", - "ip-src", - "ip-dst", - "hostname", - "domain", - "email-src", - "email-dst", - "email-subject", - "email-attachment", - "url", - "user-agent", - "AS", - "pattern-in-file", - "pattern-in-traffic", - "yara", - "attachment", - "malware-sample", - "link", - "malware-type", - "comment", - "text", - "vulnerability", - "x509-fingerprint-sha1", - "other" - ], - "Artifacts dropped": [ - "md5", - "sha1", - "sha224", - "sha256", - "sha384", - "sha512", - "sha512/224", - "sha512/256", - "ssdeep", - "imphash", - "authentihash", - "filename", - "filename|md5", - "filename|sha1", - "filename|sha224", - "filename|sha256", - "filename|sha384", - "filename|sha512", - "filename|sha512/224", - "filename|sha512/256", - "filename|authentihash", - "filename|ssdeep", - "filename|tlsh", - "filename|imphash", - "filename|pehash", - "regkey", - "regkey|value", - "pattern-in-file", - "pattern-in-memory", - "pdb", - "yara", - "attachment", - "malware-sample", - "named pipe", - "mutex", - "windows-scheduled-task", - "windows-service-name", - "windows-service-displayname", - "comment", - "text", - "x509-fingerprint-sha1", - "other" - ], - "Payload installation": [ - "md5", - "sha1", - "sha224", - "sha256", - "sha384", - "sha512", - "sha512/224", - "sha512/256", - "ssdeep", - "imphash", - "authentihash", - "pehash", - "tlsh", - "filename", - "filename|md5", - "filename|sha1", - "filename|sha224", - "filename|sha256", - "filename|sha384", - "filename|sha512", - "filename|sha512/224", - "filename|sha512/256", - "filename|authentihash", - "filename|ssdeep", - "filename|tlsh", - "filename|imphash", - "filename|pehash", - "pattern-in-file", - "pattern-in-traffic", - "pattern-in-memory", - "yara", - "vulnerability", - "attachment", - "malware-sample", - "malware-type", - "comment", - "text", - "x509-fingerprint-sha1", - "other" - ], - "Persistence mechanism": [ - "filename", - "regkey", - "regkey|value", - "comment", - "text", - "other" - ], - "Network activity": [ - "ip-src", - "ip-dst", - "hostname", - "domain", - "domain|ip", - "email-dst", - "url", - "uri", - "user-agent", - "http-method", - "AS", - "snort", - "pattern-in-file", - "pattern-in-traffic", - "attachment", - "comment", - "text", - "x509-fingerprint-sha1", - "other" - ], - "Payload type": [ - "comment", - "text", - "other" - ], - "Attribution": [ - "threat-actor", - "campaign-name", - "campaign-id", - "whois-registrant-phone", - "whois-registrant-email", - "whois-registrant-name", - "whois-registrar", - "whois-creation-date", - "comment", - "text", - "x509-fingerprint-sha1", - "other" - ], - "External analysis": [ - "md5", - "sha1", - "sha256", - "filename", - "filename|md5", - "filename|sha1", - "filename|sha256", - "ip-src", - "ip-dst", - "hostname", - "domain", - "domain|ip", - "url", - "user-agent", - "regkey", - "regkey|value", - "AS", - "snort", - "pattern-in-file", - "pattern-in-traffic", - "pattern-in-memory", - "vulnerability", - "attachment", - "malware-sample", - "link", - "comment", - "text", - "x509-fingerprint-sha1", - "other" - ], - "Financial fraud": [ - "btc", - "iban", - "bic", - "bank-account-nr", - "aba-rtn", - "bin", - "cc-number", - "prtn", - "comment", - "text", - "other" - ], - "Other": [ - "comment", - "text", - "other" - ] - } - } -} diff --git a/tests/test_offline.py b/tests/test_offline.py index 0dde96a..670195b 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -4,9 +4,14 @@ import unittest import requests_mock import json +import os import pymisp as pm from pymisp import PyMISP +from pymisp import NewEventError +from pymisp import MISPEvent +from pymisp import EncodeUpdate +from pymisp import EncodeFull @requests_mock.Mocker() @@ -18,7 +23,8 @@ class TestOffline(unittest.TestCase): self.key = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' self.event = {'Event': json.load(open('tests/misp_event.json', 'r'))} self.new_misp_event = {'Event': json.load(open('tests/new_misp_event.json', 'r'))} - self.types = json.load(open('tests/describeTypes.json', 'r')) + self.ressources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), '../pymisp/data') + self.types = json.load(open(os.path.join(self.ressources_path, 'describeTypes.json'), 'r')) self.sharing_groups = json.load(open('tests/sharing_groups.json', 'r')) self.auth_error_msg = {"name": "Authentication failed. Please make sure you pass the API key of an API enabled user along in the Authorization header.", "message": "Authentication failed. Please make sure you pass the API key of an API enabled user along in the Authorization header.", @@ -70,10 +76,10 @@ class TestOffline(unittest.TestCase): def test_publish(self, m): self.initURI(m) pymisp = PyMISP(self.domain, self.key) - e = pymisp.publish(self.event) + e = pymisp.publish(self.event) # requests-mock always return the non-published event pub = self.event pub['Event']['published'] = True - self.assertEqual(e, pub) + # self.assertEqual(e, pub) FIXME: broken test, not-published event returned e = pymisp.publish(self.event) self.assertEqual(e, {'error': 'Already published'}) @@ -104,12 +110,6 @@ class TestOffline(unittest.TestCase): error_empty_info_flatten = {u'message': u'The event could not be saved.', u'name': u'Add event failed.', u'errors': [u"Error in info: Info cannot be empty."], u'url': u'/events/add'} self.initURI(m) pymisp = PyMISP(self.domain, self.key) - with self.assertRaises(pm.api.NewEventError): - pymisp.new_event() - with self.assertRaises(pm.api.NewEventError): - pymisp.new_event(0) - with self.assertRaises(pm.api.NewEventError): - pymisp.new_event(0, 1) m.register_uri('POST', self.domain + 'events', json=error_empty_info) response = pymisp.new_event(0, 1, 0) self.assertEqual(response, error_empty_info_flatten) @@ -117,6 +117,13 @@ class TestOffline(unittest.TestCase): response = pymisp.new_event(0, 1, 0, "This is a test.", '2016-08-26', False) self.assertEqual(response, self.new_misp_event) + def test_eventObject(self, m): + self.initURI(m) + pymisp = PyMISP(self.domain, self.key) + misp_event = MISPEvent(pymisp.describe_types['result']) + misp_event.load(open('tests/57c4445b-c548-4654-af0b-4be3950d210f.json', 'r').read()) + json.dumps(misp_event, cls=EncodeUpdate) + json.dumps(misp_event, cls=EncodeFull) if __name__ == '__main__': unittest.main() From 6482a218340d8234a1c3a4dead4fae6ae5ffad45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 28 Sep 2016 18:20:37 +0200 Subject: [PATCH 126/223] More cleanup --- pymisp/api.py | 53 +++++++--------- pymisp/mispevent.py | 143 +++++++++++++++++++++--------------------- tests/test_offline.py | 3 +- 3 files changed, 97 insertions(+), 102 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 6fc41e1..d00e5fb 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -23,9 +23,10 @@ except ImportError: HAVE_REQUESTS = False from . import __version__ -from .exceptions import PyMISPError, NewAttributeError, SearchError, MissingDependency, NoURL, NoKey +from .exceptions import PyMISPError, SearchError, MissingDependency, NoURL, NoKey from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate + # Least dirty way to support python 2 and 3 try: basestring @@ -297,10 +298,10 @@ class PyMISP(object): return json.dumps(misp_event, cls=EncodeUpdate) def _prepare_full_attribute(self, category, type_value, value, to_ids, comment=None, distribution=5): - misp_attribute = MISPAttribute(self.categories, self.types, self.category_type_mapping) + misp_attribute = MISPAttribute(self.describe_types['result']) misp_attribute.set_all_values(type=type_value, value=value, category=category, to_ids=to_ids, comment=comment, distribution=distribution) - return json.dumps(misp_attribute, cls=EncodeUpdate) + return misp_attribute def _one_or_more(self, value): """Returns a list/tuple of one or more items, regardless of input.""" @@ -356,19 +357,14 @@ class PyMISP(object): else: e = MISPEvent(self.describe_types['result']) e.load(event) - for a in e.attributes: - if a.get('distribution') is None: - a['distribution'] = 5 + e.attributes += attributes response = self.update_event(event['Event']['id'], json.dumps(e, cls=EncodeUpdate)) return response - def add_named_attribute(self, event, category, type_value, value, to_ids=False, comment=None, distribution=None, proposal=False): + def add_named_attribute(self, event, type_value, value, category=None, to_ids=False, comment=None, distribution=None, proposal=False): attributes = [] - if value and category and type: - try: - attributes.append(self._prepare_full_attribute(category, type_value, value, to_ids, comment, distribution)) - except NewAttributeError as e: - return e + for value in self._one_or_more(value): + attributes.append(self._prepare_full_attribute(category, type_value, value, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) def add_hashes(self, event, category='Artifacts dropped', filename=None, md5=None, sha1=None, sha256=None, ssdeep=None, comment=None, to_ids=True, distribution=None, proposal=False): @@ -632,8 +628,8 @@ class PyMISP(object): # ######### Upload samples through the API ######### # ################################################## - def prepare_attribute(self, event_id, distribution, to_ids, category, comment, info, - analysis, threat_level_id): + def _prepare_upload(self, event_id, distribution, to_ids, category, comment, info, + analysis, threat_level_id): to_post = {'request': {}} if event_id is not None: @@ -647,12 +643,13 @@ class PyMISP(object): else: to_post['request']['event_id'] = int(event_id) - if to_ids not in [True, False]: - raise NewAttributeError('{} is invalid, to_ids has to be True or False'.format(to_ids)) + default_values = self.sane_default['malware-sample'] + if to_ids is None or not isinstance(to_ids, bool): + to_ids = bool(int(default_values['to_ids'])) to_post['request']['to_ids'] = to_ids - if category not in self.categories: - raise NewAttributeError('{} is invalid, category has to be in {}'.format(category, (', '.join(self.categories)))) + if category is None or category not in self.categories: + category = default_values['default_category'] to_post['request']['category'] = category to_post['request']['comment'] = comment @@ -665,16 +662,16 @@ class PyMISP(object): def upload_sample(self, filename, filepath, event_id, distribution=None, to_ids=True, category=None, comment=None, info=None, analysis=None, threat_level_id=None): - to_post = self.prepare_attribute(event_id, distribution, to_ids, category, - comment, info, analysis, threat_level_id) + to_post = self._prepare_upload(event_id, distribution, to_ids, category, + comment, info, analysis, threat_level_id) to_post['request']['files'] = [{'filename': filename, 'data': self._encode_file_to_upload(filepath)}] return self._upload_sample(to_post) def upload_samplelist(self, filepaths, event_id, distribution=None, to_ids=True, category=None, info=None, analysis=None, threat_level_id=None): - to_post = self.prepare_attribute(event_id, distribution, to_ids, category, - info, analysis, threat_level_id) + to_post = self._prepare_upload(event_id, distribution, to_ids, category, + info, analysis, threat_level_id) files = [] for path in filepaths: if not os.path.isfile(path): @@ -694,18 +691,14 @@ class PyMISP(object): # ############################ def __query_proposal(self, session, path, id, attribute=None): - path = path.strip('/') url = urljoin(self.root_url, 'shadow_attributes/{}/{}'.format(path, id)) - query = None if path in ['add', 'edit']: query = {'request': {'ShadowAttribute': attribute}} - if path == 'view': + response = session.post(url, data=json.dumps(query)) + elif path == 'view': response = session.get(url) - else: - if query is not None: - response = session.post(url, data=json.dumps(query)) - else: - response = session.post(url) + else: # accept or discard + response = session.post(url) return self._check_response(response) def proposal_view(self, event_id=None, proposal_id=None): diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 4e9d2b6..7697dcb 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -21,10 +21,11 @@ from .exceptions import PyMISPError, NewEventError, NewAttributeError class MISPAttribute(object): - def __init__(self, categories, types, category_type_mapping): - self.categories = categories - self.types = types - self.category_type_mapping = category_type_mapping + def __init__(self, describe_types): + self.categories = describe_types['categories'] + self.types = describe_types['types'] + self.category_type_mapping = describe_types['category_type_mappings'] + self.sane_default = describe_types['sane_defaults'] self._reinitialize_attribute() def _reinitialize_attribute(self): @@ -46,44 +47,59 @@ class MISPAttribute(object): self.ShadowAttribute = [] def set_all_values(self, **kwargs): + if kwargs.get('type') and kwargs.get('category'): + if kwargs['type'] not in self.category_type_mapping[kwargs['category']]: + raise NewAttributeError('{} and {} is an invalid combinaison, type for this category has to be in {}'.capitalizeformat(self.type, self.category, (', '.join(self.category_type_mapping[self.category])))) + # Required + if kwargs.get('type'): + self.type = kwargs['type'] + if self.type not in self.types: + raise NewAttributeError('{} is invalid, type has to be in {}'.format(self.type, (', '.join(self.types)))) + else: + raise NewAttributeError('The type of the attribute is required.') + + type_defaults = self.sane_default[self.type] + + if kwargs.get('value'): + self.value = kwargs['value'] + else: + raise NewAttributeError('The value of the attribute is required.') + # Default values - if kwargs.get('category', None): + if kwargs.get('category'): self.category = kwargs['category'] if self.category not in self.categories: raise NewAttributeError('{} is invalid, category has to be in {}'.format(self.category, (', '.join(self.categories)))) - if kwargs.get('type', None): - self.type = kwargs['type'] - if self.type not in self.types: - raise NewAttributeError('{} is invalid, type_value has to be in {}'.format(self.type, (', '.join(self.types)))) - if self.type not in self.category_type_mapping[self.category]: - raise NewAttributeError('{} and {} is an invalid combinaison, type_value for this category has to be in {}'.capitalizeformat(self.type, self.category, (', '.join(self.category_type_mapping[self.category])))) - if kwargs.get('value', None): - self.value = kwargs['value'] - if kwargs.get('to_ids', None): + else: + self.category = type_defaults['default_category'] + + if kwargs.get('to_ids'): self.to_ids = kwargs['to_ids'] if not isinstance(self.to_ids, bool): raise NewAttributeError('{} is invalid, to_ids has to be True or False'.format(self.to_ids)) - if kwargs.get('comment', None): + else: + self.to_ids = bool(int(type_defaults['to_ids'])) + if kwargs.get('comment'): self.comment = kwargs['comment'] - if kwargs.get('distribution', None): + if kwargs.get('distribution'): self.distribution = int(kwargs['distribution']) if self.distribution not in [0, 1, 2, 3, 5]: raise NewAttributeError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 5'.format(self.distribution)) # other possible values - if kwargs.get('id', None): + if kwargs.get('id'): self.id = int(kwargs['id']) - if kwargs.get('uuid', None): + if kwargs.get('uuid'): self.uuid = kwargs['uuid'] - if kwargs.get('timestamp', None): + if kwargs.get('timestamp'): self.timestamp = datetime.datetime.fromtimestamp(int(kwargs['timestamp'])) - if kwargs.get('sharing_group_id', None): + if kwargs.get('sharing_group_id'): self.sharing_group_id = int(kwargs['sharing_group_id']) - if kwargs.get('deleted', None): + if kwargs.get('deleted'): self.deleted = kwargs['deleted'] - if kwargs.get('SharingGroup', None): + if kwargs.get('SharingGroup'): self.SharingGroup = kwargs['SharingGroup'] - if kwargs.get('ShadowAttribute', None): + if kwargs.get('ShadowAttribute'): self.ShadowAttribute = kwargs['ShadowAttribute'] def _json(self): @@ -136,6 +152,7 @@ class MISPEvent(object): if not describe_types: t = json.load(open(os.path.join(self.ressources_path, 'describeTypes.json'), 'r')) describe_types = t['result'] + self.describe_types = describe_types self.categories = describe_types['categories'] self.types = describe_types['types'] self.category_type_mapping = describe_types['category_type_mappings'] @@ -150,7 +167,7 @@ class MISPEvent(object): self.distribution = 3 self.threat_level_id = 2 self.analysis = 0 - self.info = '' + self.info = None self.published = False self.date = datetime.date.today() self.attributes = [] @@ -191,24 +208,28 @@ class MISPEvent(object): self.set_all_values(**e) def set_all_values(self, **kwargs): + # Required value + if kwargs.get('info'): + self.info = kwargs['info'] + else: + raise NewAttributeError('The info field of the new event is required.') + # Default values for a valid event to send to a MISP instance - if kwargs.get('distribution', None) is not None: + if kwargs.get('distribution') is not None: self.distribution = int(kwargs['distribution']) if self.distribution not in [0, 1, 2, 3]: raise NewEventError('{} is invalid, the distribution has to be in 0, 1, 2, 3'.format(self.distribution)) - if kwargs.get('threat_level_id', None) is not None: + if kwargs.get('threat_level_id') is not None: self.threat_level_id = int(kwargs['threat_level_id']) if self.threat_level_id not in [1, 2, 3, 4]: raise NewEventError('{} is invalid, the threat_level has to be in 1, 2, 3, 4'.format(self.threat_level_id)) - if kwargs.get('analysis', None) is not None: + if kwargs.get('analysis') is not None: self.analysis = int(kwargs['analysis']) if self.analysis not in [0, 1, 2]: raise NewEventError('{} is invalid, the analysis has to be in 0, 1, 2'.format(self.analysis)) - if kwargs.get('info', None): - self.info = kwargs['info'] - if kwargs.get('published', None) is not None: + if kwargs.get('published') is not None: self.publish() - if kwargs.get('date', None): + if kwargs.get('date'): if isinstance(kwargs['date'], str): self.date = parse(kwargs['date']) elif isinstance(kwargs['date'], datetime.datetime): @@ -217,42 +238,42 @@ class MISPEvent(object): self.date = kwargs['date'] else: raise NewEventError('Invalid format for the date: {} - {}'.format(kwargs['date'], type(kwargs['date']))) - if kwargs.get('Attribute', None): + if kwargs.get('Attribute'): for a in kwargs['Attribute']: - attribute = MISPAttribute(self.categories, self.types, self.category_type_mapping) + attribute = MISPAttribute(self.describe_types) attribute.set_all_values(**a) self.attributes.append(attribute) # All other keys - if kwargs.get('id', None): + if kwargs.get('id'): self.id = int(kwargs['id']) - if kwargs.get('orgc_id', None): + if kwargs.get('orgc_id'): self.orgc_id = int(kwargs['orgc_id']) - if kwargs.get('org_id', None): + if kwargs.get('org_id'): self.org_id = int(kwargs['org_id']) - if kwargs.get('uuid', None): + if kwargs.get('uuid'): self.uuid = kwargs['uuid'] - if kwargs.get('attribute_count', None): + if kwargs.get('attribute_count'): self.attribute_count = int(kwargs['attribute_count']) - if kwargs.get('timestamp', None): + if kwargs.get('timestamp'): self.timestamp = datetime.datetime.fromtimestamp(int(kwargs['timestamp'])) - if kwargs.get('proposal_email_lock', None): + if kwargs.get('proposal_email_lock'): self.proposal_email_lock = kwargs['proposal_email_lock'] - if kwargs.get('locked', None): + if kwargs.get('locked'): self.locked = kwargs['locked'] - if kwargs.get('publish_timestamp', None): + if kwargs.get('publish_timestamp'): self.publish_timestamp = datetime.datetime.fromtimestamp(int(kwargs['publish_timestamp'])) - if kwargs.get('sharing_group_id', None): + if kwargs.get('sharing_group_id'): self.sharing_group_id = int(kwargs['sharing_group_id']) - if kwargs.get('Org', None): + if kwargs.get('Org'): self.Org = kwargs['Org'] - if kwargs.get('Orgc', None): + if kwargs.get('Orgc'): self.Orgc = kwargs['Orgc'] - if kwargs.get('ShadowAttribute', None): + if kwargs.get('ShadowAttribute'): self.ShadowAttribute = kwargs['ShadowAttribute'] - if kwargs.get('RelatedEvent', None): + if kwargs.get('RelatedEvent'): self.RelatedEvent = kwargs['RelatedEvent'] - if kwargs.get('Tag', None): + if kwargs.get('Tag'): self.Tag = kwargs['Tag'] def _json(self): @@ -309,27 +330,7 @@ class MISPEvent(object): def unpublish(self): self.published = False - def add_attribute(self, type_value, value, **kwargs): - if not self.sane_default.get(type_value): - raise NewAttributeError("{} is an invalid type. Can only be one of the following: {}".format(type_value, ', '.join(self.types))) - defaults = self.sane_default[type_value] - if kwargs.get('category'): - category = kwargs.get('category') - else: - category = defaults['default_category'] - if kwargs.get('to_ids'): - to_ids = bool(int(kwargs.get('to_ids'))) - else: - to_ids = bool(int(defaults['to_ids'])) - if kwargs.get('comment'): - comment = kwargs.get('comment') - else: - comment = None - if kwargs.get('distribution'): - distribution = int(kwargs.get('distribution')) - else: - distribution = 5 - attribute = MISPAttribute(self.categories, self.types, self.category_type_mapping) - attribute.set_all_values(type=type_value, value=value, category=category, - to_ids=to_ids, comment=comment, distribution=distribution) + def add_attribute(self, type, value, **kwargs): + attribute = MISPAttribute(self.describe_types) + attribute.set_all_values(type=type, value=value, **kwargs) self.attributes.append(attribute) diff --git a/tests/test_offline.py b/tests/test_offline.py index 670195b..b08d5c3 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -111,7 +111,8 @@ class TestOffline(unittest.TestCase): self.initURI(m) pymisp = PyMISP(self.domain, self.key) m.register_uri('POST', self.domain + 'events', json=error_empty_info) - response = pymisp.new_event(0, 1, 0) + # TODO Add test exception if info field isn't set + response = pymisp.new_event(0, 1, 0, 'Foo') self.assertEqual(response, error_empty_info_flatten) m.register_uri('POST', self.domain + 'events', json=self.new_misp_event) response = pymisp.new_event(0, 1, 0, "This is a test.", '2016-08-26', False) From dcd76ec5c4ffba2335d42068a3dc3b7fd21646d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 28 Sep 2016 18:50:05 +0200 Subject: [PATCH 127/223] Fix python 2.7 support, add missing test file --- pymisp/mispevent.py | 10 ++++++++-- tests/57c4445b-c548-4654-af0b-4be3950d210f.json | 1 + 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 tests/57c4445b-c548-4654-af0b-4be3950d210f.json diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 7697dcb..4812ac0 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -18,6 +18,12 @@ except ImportError: from .exceptions import PyMISPError, NewEventError, NewAttributeError +# Least dirty way to support python 2 and 3 +try: + basestring +except NameError: + basestring = str + class MISPAttribute(object): @@ -192,7 +198,7 @@ class MISPEvent(object): def load(self, json_event): self.new = False self.dump_full = True - if isinstance(json_event, str): + if isinstance(json_event, basestring): loaded = json.loads(json_event) if loaded.get('response'): event = loaded.get('response')[0] @@ -230,7 +236,7 @@ class MISPEvent(object): if kwargs.get('published') is not None: self.publish() if kwargs.get('date'): - if isinstance(kwargs['date'], str): + if isinstance(kwargs['date'], basestring): self.date = parse(kwargs['date']) elif isinstance(kwargs['date'], datetime.datetime): self.date = kwargs['date'].date() diff --git a/tests/57c4445b-c548-4654-af0b-4be3950d210f.json b/tests/57c4445b-c548-4654-af0b-4be3950d210f.json new file mode 100644 index 0000000..29bb02c --- /dev/null +++ b/tests/57c4445b-c548-4654-af0b-4be3950d210f.json @@ -0,0 +1 @@ +{"Event": {"info": "Ransomware - Xorist", "publish_timestamp": "1472548231", "timestamp": "1472541011", "analysis": "2", "Attribute": [{"category": "External analysis", "comment": "Imported via the Freetext Import Tool - Xchecked via VT: b3c4ae251f8094fa15b510051835c657eaef2a6cea46075d3aec964b14a99f68", "uuid": "57c5300c-0560-4146-bfaa-40e802de0b81", "timestamp": "1472540684", "to_ids": false, "value": "https://www.virustotal.com/file/b3c4ae251f8094fa15b510051835c657eaef2a6cea46075d3aec964b14a99f68/analysis/1469554268/", "type": "link"}, {"category": "External analysis", "comment": "", "uuid": "57c5310b-dc34-43cb-8b8e-4846950d210f", "timestamp": "1472541011", "to_ids": false, "value": "http://www.xylibox.com/2011/06/have-fun-with-trojan-ransomwin32xorist.html", "type": "link"}, {"category": "Other", "comment": "", "uuid": "57c444c0-8004-48fa-9c33-8aca950d210f", "timestamp": "1472480448", "to_ids": false, "value": "UPX packed", "type": "comment"}, {"category": "Other", "comment": "", "uuid": "57c44648-96f4-45d4-a8eb-453e950d210f", "timestamp": "1472480840", "to_ids": false, "value": "Key: 85350044dF4AC3518D185678A9414A7F,\r\nEncryption rounds:8,\r\nStart offset: 64,\r\nAlgorithm: TEA", "type": "text"}, {"category": "Payload delivery", "comment": "Imported via the Freetext Import Tool", "uuid": "57c4448a-fb04-457d-87e7-4127950d210f", "timestamp": "1472480394", "to_ids": true, "value": "3Z4wnG9603it23y.exe", "type": "filename"}, {"category": "Payload delivery", "comment": "Imported via the Freetext Import Tool", "uuid": "57c4448b-454c-4d17-90d1-4d2f950d210f", "timestamp": "1472480395", "to_ids": true, "value": "0749bae92ca336a02c83d126e04ec628", "type": "md5"}, {"category": "Payload delivery", "comment": "Imported via the Freetext Import Tool", "uuid": "57c4448a-bef0-4ba7-a071-444e950d210f", "timestamp": "1472480394", "to_ids": true, "value": "77b0c41b7d340b8a3d903f21347bbf06aa766b5b", "type": "sha1"}, {"category": "Payload delivery", "comment": "Imported via the Freetext Import Tool", "uuid": "57c4448b-3fa4-4d65-9ccc-4afa950d210f", "timestamp": "1472480395", "to_ids": true, "value": "b3c4ae251f8094fa15b510051835c657eaef2a6cea46075d3aec964b14a99f68", "type": "sha256"}, {"category": "Persistence mechanism", "comment": "", "uuid": "57c54b0f-27a4-458b-8e63-4455950d210f", "timestamp": "1472547599", "to_ids": true, "value": "Software\\Wow6432Node\\Microsoft\\Windows\\CurrentVersion\\Run|%TEMP%\\3Z4wnG9603it23y.exe", "type": "regkey|value"}], "Tag": [{"colour": "#ffffff", "exportable": true, "name": "tlp:white"}, {"colour": "#3d7a00", "exportable": true, "name": "circl:incident-classification=\"malware\""}, {"colour": "#420053", "exportable": true, "name": "ms-caro-malware:malware-type=\"Ransom\""}, {"colour": "#2c4f00", "exportable": true, "name": "malware_classification:malware-category=\"Ransomware\""}], "published": true, "date": "2016-08-29", "Orgc": {"name": "CIRCL", "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f"}, "threat_level_id": "3", "uuid": "57c4445b-c548-4654-af0b-4be3950d210f"}} \ No newline at end of file From 5ee23d46c04bed4cb5292cba2fecb4286c88f5d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 30 Sep 2016 16:06:41 +0200 Subject: [PATCH 128/223] Make sure all integers are string in the dumped json --- pymisp/data/schema-lax.json | 2 +- pymisp/data/schema.json | 38 ++++++++++++++++++------------------- pymisp/mispevent.py | 14 +++++++++++++- 3 files changed, 33 insertions(+), 21 deletions(-) diff --git a/pymisp/data/schema-lax.json b/pymisp/data/schema-lax.json index 09bc780..f21ac64 100644 --- a/pymisp/data/schema-lax.json +++ b/pymisp/data/schema-lax.json @@ -225,7 +225,7 @@ }, "Event": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event", - "type": "object", + "type": "array", "items": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0", "type": "object", diff --git a/pymisp/data/schema.json b/pymisp/data/schema.json index 4560498..d7cb964 100644 --- a/pymisp/data/schema.json +++ b/pymisp/data/schema.json @@ -9,15 +9,15 @@ "properties": { "id": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/id", - "type": "integer" + "type": "string" }, "orgc_id": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/orgc_id", - "type": "integer" + "type": "string" }, "org_id": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/org_id", - "type": "integer" + "type": "string" }, "date": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/date", @@ -25,7 +25,7 @@ }, "threat_level_id": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/threat_level_id", - "type": "integer" + "type": "string" }, "info": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/info", @@ -41,19 +41,19 @@ }, "attribute_count": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/attribute_count", - "type": "integer" + "type": "string" }, "analysis": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/analysis", - "type": "integer" + "type": "string" }, "timestamp": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/timestamp", - "type": "integer" + "type": "string" }, "distribution": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/distribution", - "type": "integer" + "type": "string" }, "proposal_email_lock": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/proposal_email_lock", @@ -65,11 +65,11 @@ }, "publish_timestamp": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/publish_timestamp", - "type": "integer" + "type": "string" }, "sharing_group_id": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/sharing_group_id", - "type": "integer" + "type": "string" }, "Org": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Org", @@ -136,15 +136,15 @@ }, "event_id": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17/event_id", - "type": "integer" + "type": "string" }, "distribution": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17/distribution", - "type": "integer" + "type": "string" }, "timestamp": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17/timestamp", - "type": "integer" + "type": "string" }, "comment": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17/comment", @@ -152,7 +152,7 @@ }, "sharing_group_id": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17/sharing_group_id", - "type": "integer" + "type": "string" }, "value": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/Attribute/17/value", @@ -225,7 +225,7 @@ }, "Event": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event", - "type": "object", + "type": "array", "items": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0", "type": "object", @@ -240,7 +240,7 @@ }, "threat_level_id": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0/threat_level_id", - "type": "integer" + "type": "string" }, "info": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0/info", @@ -260,7 +260,7 @@ }, "timestamp": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0/timestamp", - "type": "integer" + "type": "string" }, "distribution": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0/distribution", @@ -268,11 +268,11 @@ }, "org_id": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0/org_id", - "type": "integer" + "type": "string" }, "orgc_id": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0/orgc_id", - "type": "integer" + "type": "string" } } }, diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 4812ac0..1320113 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -114,6 +114,7 @@ class MISPAttribute(object): 'comment': self.comment} if self.sharing_group_id: to_return['sharing_group_id'] = self.sharing_group_id + to_return = _int_to_str(to_return) return to_return def _json_full(self): @@ -130,6 +131,7 @@ class MISPAttribute(object): to_return['ShadowAttribute'] = self.ShadowAttribute if self.SharingGroup: to_return['SharingGroup'] = self.SharingGroup + to_return = _int_to_str(to_return) return to_return @@ -149,6 +151,14 @@ class EncodeFull(JSONEncoder): return JSONEncoder.default(self, obj) +def _int_to_str(d): + # transform all integer back to string + for k, v in d.items(): + if isinstance(v, int) and not isinstance(v, bool): + d[k] = str(v) + return d + + class MISPEvent(object): def __init__(self, describe_types=None): @@ -300,6 +310,7 @@ class MISPEvent(object): to_return['Event']['sharing_group_id'] = self.sharing_group_id if self.Tag: to_return['Event']['Tag'] = self.Tag + to_return['Event'] = _int_to_str(to_return['Event']) to_return['Event']['Attribute'] = [a._json() for a in self.attributes] jsonschema.validate(to_return, self.json_schema) return to_return @@ -308,7 +319,7 @@ class MISPEvent(object): to_return = self._json() if self.locked is not None: to_return['Event']['locked'] = self.locked - if self.attribute_count: + if self.attribute_count is not None: to_return['Event']['attribute_count'] = self.attribute_count if self.RelatedEvent: to_return['Event']['RelatedEvent'] = self.RelatedEvent @@ -326,6 +337,7 @@ class MISPEvent(object): to_return['Event']['publish_timestamp'] = int(time.mktime(self.publish_timestamp.timetuple())) if self.timestamp: to_return['Event']['timestamp'] = int(time.mktime(self.timestamp.timetuple())) + to_return['Event'] = _int_to_str(to_return['Event']) to_return['Event']['Attribute'] = [a._json_full() for a in self.attributes] jsonschema.validate(to_return, self.json_schema) return to_return From 8a931a89f314c7e4d98c58f7169cb65c1b6b9f2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 5 Oct 2016 11:07:40 +0200 Subject: [PATCH 129/223] Fix upload function --- examples/upload.py | 9 +++++---- pymisp/api.py | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/examples/upload.py b/examples/upload.py index 4f49f20..4c6708d 100755 --- a/examples/upload.py +++ b/examples/upload.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- from pymisp import PyMISP -from keys import misp_url, misp_key,misp_verifycert +from keys import misp_url, misp_key, misp_verifycert import argparse import os import glob @@ -12,8 +12,8 @@ def init(url, key): return PyMISP(url, key, misp_verifycert, 'json') -def upload_files(m, eid, paths, distrib, ids, categ, info, analysis, threat): - out = m.upload_samplelist(paths, eid, distrib, ids, categ, info, analysis, threat) +def upload_files(m, eid, paths, distrib, ids, categ, comment, info, analysis, threat): + out = m.upload_samplelist(paths, eid, distrib, ids, categ, comment, info, analysis, threat) print(out) if __name__ == '__main__': @@ -26,6 +26,7 @@ if __name__ == '__main__': parser.add_argument("-i", "--info", help="Used to populate the event info field if no event ID supplied.") parser.add_argument("-a", "--analysis", type=int, help="The analysis level of the newly created event, if applicatble. [0-2]") parser.add_argument("-t", "--threat", type=int, help="The threat level ID of the newly created event, if applicatble. [1-4]") + parser.add_argument("-co", "--comment", type=str, help="Comment for the uploaded file(s).") args = parser.parse_args() misp = init(misp_url, misp_key) @@ -39,4 +40,4 @@ if __name__ == '__main__': print('invalid file') exit(0) - upload_files(misp, args.event, files, args.distrib, args.ids, args.categ, args.info, args.analysis, args.threat) + upload_files(misp, args.event, files, args.distrib, args.ids, args.categ, args.comment, args.info, args.analysis, args.threat) diff --git a/pymisp/api.py b/pymisp/api.py index d00e5fb..6d7d92f 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -668,10 +668,10 @@ class PyMISP(object): return self._upload_sample(to_post) def upload_samplelist(self, filepaths, event_id, distribution=None, - to_ids=True, category=None, info=None, + to_ids=True, category=None, comment=None, info=None, analysis=None, threat_level_id=None): to_post = self._prepare_upload(event_id, distribution, to_ids, category, - info, analysis, threat_level_id) + comment, info, analysis, threat_level_id) files = [] for path in filepaths: if not os.path.isfile(path): From e91ad63ccf340e99858793a3c9d0fdca329197ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 5 Oct 2016 11:43:33 +0200 Subject: [PATCH 130/223] More logical output for _prepare_full_event --- pymisp/api.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 6d7d92f..b1b3af6 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -295,7 +295,7 @@ class PyMISP(object): analysis=analysis, date=date) if published: misp_event.publish() - return json.dumps(misp_event, cls=EncodeUpdate) + return misp_event def _prepare_full_attribute(self, category, type_value, value, to_ids, comment=None, distribution=5): misp_attribute = MISPAttribute(self.describe_types['result']) @@ -334,8 +334,8 @@ class PyMISP(object): return self.update_event(event['Event']['id'], json.dumps(e, cls=EncodeUpdate)) def new_event(self, distribution=None, threat_level_id=None, analysis=None, info=None, date=None, published=False): - data = self._prepare_full_event(distribution, threat_level_id, analysis, info, date, published) - return self.add_event(data) + misp_event = self._prepare_full_event(distribution, threat_level_id, analysis, info, date, published) + return self.add_event(json.dumps(misp_event, cls=EncodeUpdate)) def add_tag(self, event, tag): session = self.__prepare_session() @@ -639,7 +639,11 @@ class PyMISP(object): pass if not isinstance(event_id, int): # New event - to_post['request'] = self._prepare_full_event(distribution, threat_level_id, analysis, info) + misp_event = self._prepare_full_event(distribution, threat_level_id, analysis, info) + to_post['request']['distribution'] = misp_event.distribution + to_post['request']['info'] = misp_event.info + to_post['request']['analysis'] = misp_event.analysis + to_post['request']['threat_level_id'] = misp_event.threat_level_id else: to_post['request']['event_id'] = int(event_id) From b6072f7145863bd9f937d17a0878107c0b713ca2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 7 Oct 2016 16:50:57 +0200 Subject: [PATCH 131/223] Raise exception if the dest instance is < 2.4.52, set User-Agent. --- pymisp/api.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index b1b3af6..3cd68f8 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -112,11 +112,11 @@ class PyMISP(object): self.categories = self.describe_types['result']['categories'] self.types = self.describe_types['result']['types'] self.category_type_mapping = self.describe_types['result']['category_type_mappings'] - # New in 2.5.52 if self.describe_types['result'].get('sane_defaults'): + # New in 2.5.52 self.sane_default = self.describe_types['result']['sane_defaults'] else: - self.sane_default = {} + raise PyMISPError('The MISP server your are trying to reach is outdated (<2.4.52). Please use PyMISP v2.4.51.1 (pip install -I PyMISP==v2.4.51.1) and/or contact your administrator.') def __prepare_session(self, output='json'): """ @@ -131,7 +131,8 @@ class PyMISP(object): session.headers.update( {'Authorization': self.key, 'Accept': 'application/{}'.format(output), - 'content-type': 'application/{}'.format(output)}) + 'content-type': 'application/{}'.format(output), + 'User-Agent': 'PyMISP {}'.format(__version__)}) return session def flatten_error_messages(self, response): From 8817674dd06b088cde751c4c4d1e2400e0a9f528 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 10 Oct 2016 12:23:03 +0200 Subject: [PATCH 132/223] Do not dump an empty list of attrbutes. --- pymisp/mispevent.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 1320113..678b084 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -311,7 +311,8 @@ class MISPEvent(object): if self.Tag: to_return['Event']['Tag'] = self.Tag to_return['Event'] = _int_to_str(to_return['Event']) - to_return['Event']['Attribute'] = [a._json() for a in self.attributes] + if self.attributes: + to_return['Event']['Attribute'] = [a._json() for a in self.attributes] jsonschema.validate(to_return, self.json_schema) return to_return @@ -338,7 +339,8 @@ class MISPEvent(object): if self.timestamp: to_return['Event']['timestamp'] = int(time.mktime(self.timestamp.timetuple())) to_return['Event'] = _int_to_str(to_return['Event']) - to_return['Event']['Attribute'] = [a._json_full() for a in self.attributes] + if self.attributes: + to_return['Event']['Attribute'] = [a._json_full() for a in self.attributes] jsonschema.validate(to_return, self.json_schema) return to_return From 78919272ab579b16592a12ddae9528dfb9904a8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 10 Oct 2016 12:24:17 +0200 Subject: [PATCH 133/223] Use sane_defaults from describeTypes.json if unable to fetch it from the instance. --- pymisp/api.py | 41 +- pymisp/data/describeTypes.json | 707 ++++++++++++++++++++++++++++++++- 2 files changed, 728 insertions(+), 20 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 3cd68f8..0d01298 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -102,21 +102,24 @@ class PyMISP(object): except Exception as e: raise PyMISPError('Unable to connect to MISP ({}). Please make sure the API key and the URL are correct (http/https is required): {}'.format(self.root_url, e)) - session = self.__prepare_session() - response = session.get(urljoin(self.root_url, 'attributes/describeTypes.json')) - self.describe_types = self._check_response(response) - if self.describe_types.get('error'): - for e in self.describe_types.get('error'): - raise PyMISPError('Failed: {}'.format(e)) + try: + session = self.__prepare_session() + response = session.get(urljoin(self.root_url, 'attributes/describeTypes.json')) + describe_types = self._check_response(response) + if describe_types.get('error'): + for e in describe_types.get('error'): + raise PyMISPError('Failed: {}'.format(e)) + self.describe_types = describe_types['result'] + if not self.describe_types.get('sane_defaults'): + raise PyMISPError('The MISP server your are trying to reach is outdated (<2.4.52). Please use PyMISP v2.4.51.1 (pip install -I PyMISP==v2.4.51.1) and/or contact your administrator.') + except: + describe_types = json.load(open(os.path.join(self.ressources_path, 'describeTypes.json'), 'r')) + self.describe_types = describe_types['result'] - self.categories = self.describe_types['result']['categories'] - self.types = self.describe_types['result']['types'] - self.category_type_mapping = self.describe_types['result']['category_type_mappings'] - if self.describe_types['result'].get('sane_defaults'): - # New in 2.5.52 - self.sane_default = self.describe_types['result']['sane_defaults'] - else: - raise PyMISPError('The MISP server your are trying to reach is outdated (<2.4.52). Please use PyMISP v2.4.51.1 (pip install -I PyMISP==v2.4.51.1) and/or contact your administrator.') + self.categories = self.describe_types['categories'] + self.types = self.describe_types['types'] + self.category_type_mapping = self.describe_types['category_type_mappings'] + self.sane_default = self.describe_types['sane_defaults'] def __prepare_session(self, output='json'): """ @@ -291,7 +294,7 @@ class PyMISP(object): # ############################################## def _prepare_full_event(self, distribution, threat_level_id, analysis, info, date=None, published=False): - misp_event = MISPEvent(self.describe_types['result']) + misp_event = MISPEvent(self.describe_types) misp_event.set_all_values(info=info, distribution=distribution, threat_level_id=threat_level_id, analysis=analysis, date=date) if published: @@ -299,7 +302,7 @@ class PyMISP(object): return misp_event def _prepare_full_attribute(self, category, type_value, value, to_ids, comment=None, distribution=5): - misp_attribute = MISPAttribute(self.describe_types['result']) + misp_attribute = MISPAttribute(self.describe_types) misp_attribute.set_all_values(type=type_value, value=value, category=category, to_ids=to_ids, comment=comment, distribution=distribution) return misp_attribute @@ -323,13 +326,13 @@ class PyMISP(object): def publish(self, event): if event['Event']['published']: return {'error': 'Already published'} - e = MISPEvent(self.describe_types['result']) + e = MISPEvent(self.describe_types) e.load(event) e.publish() return self.update_event(event['Event']['id'], json.dumps(e, cls=EncodeUpdate)) def change_threat_level(self, event, threat_level_id): - e = MISPEvent(self.describe_types['result']) + e = MISPEvent(self.describe_types) e.load(event) e.threat_level_id = threat_level_id return self.update_event(event['Event']['id'], json.dumps(e, cls=EncodeUpdate)) @@ -356,7 +359,7 @@ class PyMISP(object): if proposal: response = self.proposal_add(event['Event']['id'], attributes) else: - e = MISPEvent(self.describe_types['result']) + e = MISPEvent(self.describe_types) e.load(event) e.attributes += attributes response = self.update_event(event['Event']['id'], json.dumps(e, cls=EncodeUpdate)) diff --git a/pymisp/data/describeTypes.json b/pymisp/data/describeTypes.json index 5ac2edd..820341e 100644 --- a/pymisp/data/describeTypes.json +++ b/pymisp/data/describeTypes.json @@ -1 +1,706 @@ -{"result":{"sane_defaults":{"md5":{"default_category":"Payload delivery","to_ids":1},"sha1":{"default_category":"Payload delivery","to_ids":1},"sha256":{"default_category":"Payload delivery","to_ids":1},"filename":{"default_category":"Payload delivery","to_ids":1},"pdb":{"default_category":"Artifacts dropped","to_ids":0},"filename|md5":{"default_category":"Payload delivery","to_ids":1},"filename|sha1":{"default_category":"Payload delivery","to_ids":1},"filename|sha256":{"default_category":"Payload delivery","to_ids":1},"ip-src":{"default_category":"Network activity","to_ids":1},"ip-dst":{"default_category":"Network activity","to_ids":1},"hostname":{"default_category":"Network activity","to_ids":1},"domain":{"default_category":"Network activity","to_ids":1},"domain|ip":{"default_category":"Network activity","to_ids":1},"email-src":{"default_category":"Payload delivery","to_ids":1},"email-dst":{"default_category":"Network activity","to_ids":1},"email-subject":{"default_category":"Payload delivery","to_ids":0},"email-attachment":{"default_category":"Payload delivery","to_ids":1},"url":{"default_category":"External analysis","to_ids":1},"http-method":{"default_category":"Network activity","to_ids":0},"user-agent":{"default_category":"Network activity","to_ids":0},"regkey":{"default_category":"Persistence mechanism","to_ids":1},"regkey|value":{"default_category":"Persistence mechanism","to_ids":1},"AS":{"default_category":"Network activity","to_ids":0},"snort":{"default_category":"Network activity","to_ids":1},"pattern-in-file":{"default_category":"Payload installation","to_ids":1},"pattern-in-traffic":{"default_category":"Network activity","to_ids":1},"pattern-in-memory":{"default_category":"Payload installation","to_ids":1},"yara":{"default_category":"Payload installation","to_ids":1},"vulnerability":{"default_category":"External analysis","to_ids":0},"attachment":{"default_category":"External analysis","to_ids":0},"malware-sample":{"default_category":"Payload delivery","to_ids":1},"link":{"default_category":"External analysis","to_ids":0},"comment":{"default_category":"Other","to_ids":0},"text":{"default_category":"Other","to_ids":0},"other":{"default_category":"Other","to_ids":0},"named pipe":{"default_category":"Artifacts dropped","to_ids":0},"mutex":{"default_category":"Artifacts dropped","to_ids":1},"target-user":{"default_category":"Targeting data","to_ids":0},"target-email":{"default_category":"Targeting data","to_ids":0},"target-machine":{"default_category":"Targeting data","to_ids":0},"target-org":{"default_category":"Targeting data","to_ids":0},"target-location":{"default_category":"Targeting data","to_ids":0},"target-external":{"default_category":"Targeting data","to_ids":0},"btc":{"default_category":"Financial fraud","to_ids":1},"iban":{"default_category":"Financial fraud","to_ids":1},"bic":{"default_category":"Financial fraud","to_ids":1},"bank-account-nr":{"default_category":"Financial fraud","to_ids":1},"aba-rtn":{"default_category":"Financial fraud","to_ids":1},"bin":{"default_category":"Financial fraud","to_ids":1},"cc-number":{"default_category":"Financial fraud","to_ids":1},"prtn":{"default_category":"Financial fraud","to_ids":1},"threat-actor":{"default_category":"Attribution","to_ids":0},"campaign-name":{"default_category":"Attribution","to_ids":0},"campaign-id":{"default_category":"Attribution","to_ids":0},"malware-type":{"default_category":"Payload delivery","to_ids":0},"uri":{"default_category":"Network activity","to_ids":1},"authentihash":{"default_category":"Payload delivery","to_ids":1},"ssdeep":{"default_category":"Payload delivery","to_ids":1},"imphash":{"default_category":"Payload delivery","to_ids":1},"pehash":{"default_category":"Payload delivery","to_ids":1},"sha224":{"default_category":"Payload delivery","to_ids":1},"sha384":{"default_category":"Payload delivery","to_ids":1},"sha512":{"default_category":"Payload delivery","to_ids":1},"sha512\/224":{"default_category":"Payload delivery","to_ids":1},"sha512\/256":{"default_category":"Payload delivery","to_ids":1},"tlsh":{"default_category":"Payload delivery","to_ids":1},"filename|authentihash":{"default_category":"Payload delivery","to_ids":1},"filename|ssdeep":{"default_category":"Payload delivery","to_ids":1},"filename|imphash":{"default_category":"Payload delivery","to_ids":1},"filename|pehash":{"default_category":"Payload delivery","to_ids":1},"filename|sha224":{"default_category":"Payload delivery","to_ids":1},"filename|sha384":{"default_category":"Payload delivery","to_ids":1},"filename|sha512":{"default_category":"Payload delivery","to_ids":1},"filename|sha512\/224":{"default_category":"Payload delivery","to_ids":1},"filename|sha512\/256":{"default_category":"Payload delivery","to_ids":1},"filename|tlsh":{"default_category":"Payload delivery","to_ids":1},"windows-scheduled-task":{"default_category":"Artifacts dropped","to_ids":0},"windows-service-name":{"default_category":"Artifacts dropped","to_ids":0},"windows-service-displayname":{"default_category":"Artifacts dropped","to_ids":0},"whois-registrant-email":{"default_category":"Attribution","to_ids":0},"whois-registrant-phone":{"default_category":"Attribution","to_ids":0},"whois-registrant-name":{"default_category":"Attribution","to_ids":0},"whois-registrar":{"default_category":"Attribution","to_ids":0},"whois-creation-date":{"default_category":"Attribution","to_ids":0},"x509-fingerprint-sha1":{"default_category":"Network activity","to_ids":1}},"types":["md5","sha1","sha256","filename","pdb","filename|md5","filename|sha1","filename|sha256","ip-src","ip-dst","hostname","domain","domain|ip","email-src","email-dst","email-subject","email-attachment","url","http-method","user-agent","regkey","regkey|value","AS","snort","pattern-in-file","pattern-in-traffic","pattern-in-memory","yara","vulnerability","attachment","malware-sample","link","comment","text","other","named pipe","mutex","target-user","target-email","target-machine","target-org","target-location","target-external","btc","iban","bic","bank-account-nr","aba-rtn","bin","cc-number","prtn","threat-actor","campaign-name","campaign-id","malware-type","uri","authentihash","ssdeep","imphash","pehash","sha224","sha384","sha512","sha512\/224","sha512\/256","tlsh","filename|authentihash","filename|ssdeep","filename|imphash","filename|pehash","filename|sha224","filename|sha384","filename|sha512","filename|sha512\/224","filename|sha512\/256","filename|tlsh","windows-scheduled-task","windows-service-name","windows-service-displayname","whois-registrant-email","whois-registrant-phone","whois-registrant-name","whois-registrar","whois-creation-date","x509-fingerprint-sha1"],"categories":["Internal reference","Targeting data","Antivirus detection","Payload delivery","Artifacts dropped","Payload installation","Persistence mechanism","Network activity","Payload type","Attribution","External analysis","Financial fraud","Other"],"category_type_mappings":{"Internal reference":["text","link","comment","other"],"Targeting data":["target-user","target-email","target-machine","target-org","target-location","target-external","comment"],"Antivirus detection":["link","comment","text","attachment","other"],"Payload delivery":["md5","sha1","sha224","sha256","sha384","sha512","sha512\/224","sha512\/256","ssdeep","imphash","authentihash","pehash","tlsh","filename","filename|md5","filename|sha1","filename|sha224","filename|sha256","filename|sha384","filename|sha512","filename|sha512\/224","filename|sha512\/256","filename|authentihash","filename|ssdeep","filename|tlsh","filename|imphash","filename|pehash","ip-src","ip-dst","hostname","domain","email-src","email-dst","email-subject","email-attachment","url","user-agent","AS","pattern-in-file","pattern-in-traffic","yara","attachment","malware-sample","link","malware-type","comment","text","vulnerability","x509-fingerprint-sha1","other"],"Artifacts dropped":["md5","sha1","sha224","sha256","sha384","sha512","sha512\/224","sha512\/256","ssdeep","imphash","authentihash","filename","filename|md5","filename|sha1","filename|sha224","filename|sha256","filename|sha384","filename|sha512","filename|sha512\/224","filename|sha512\/256","filename|authentihash","filename|ssdeep","filename|tlsh","filename|imphash","filename|pehash","regkey","regkey|value","pattern-in-file","pattern-in-memory","pdb","yara","attachment","malware-sample","named pipe","mutex","windows-scheduled-task","windows-service-name","windows-service-displayname","comment","text","x509-fingerprint-sha1","other"],"Payload installation":["md5","sha1","sha224","sha256","sha384","sha512","sha512\/224","sha512\/256","ssdeep","imphash","authentihash","pehash","tlsh","filename","filename|md5","filename|sha1","filename|sha224","filename|sha256","filename|sha384","filename|sha512","filename|sha512\/224","filename|sha512\/256","filename|authentihash","filename|ssdeep","filename|tlsh","filename|imphash","filename|pehash","pattern-in-file","pattern-in-traffic","pattern-in-memory","yara","vulnerability","attachment","malware-sample","malware-type","comment","text","x509-fingerprint-sha1","other"],"Persistence mechanism":["filename","regkey","regkey|value","comment","text","other"],"Network activity":["ip-src","ip-dst","hostname","domain","domain|ip","email-dst","url","uri","user-agent","http-method","AS","snort","pattern-in-file","pattern-in-traffic","attachment","comment","text","x509-fingerprint-sha1","other"],"Payload type":["comment","text","other"],"Attribution":["threat-actor","campaign-name","campaign-id","whois-registrant-phone","whois-registrant-email","whois-registrant-name","whois-registrar","whois-creation-date","comment","text","x509-fingerprint-sha1","other"],"External analysis":["md5","sha1","sha256","filename","filename|md5","filename|sha1","filename|sha256","ip-src","ip-dst","hostname","domain","domain|ip","url","user-agent","regkey","regkey|value","AS","snort","pattern-in-file","pattern-in-traffic","pattern-in-memory","vulnerability","attachment","malware-sample","link","comment","text","x509-fingerprint-sha1","other"],"Financial fraud":["btc","iban","bic","bank-account-nr","aba-rtn","bin","cc-number","prtn","comment","text","other"],"Other":["comment","text","other"]}}} \ No newline at end of file +{ + "result": { + "sane_defaults": { + "md5": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "sha1": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "sha256": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "pdb": { + "default_category": "Artifacts dropped", + "to_ids": 0 + }, + "filename|md5": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha1": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha256": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "ip-src": { + "default_category": "Network activity", + "to_ids": 1 + }, + "ip-dst": { + "default_category": "Network activity", + "to_ids": 1 + }, + "hostname": { + "default_category": "Network activity", + "to_ids": 1 + }, + "domain": { + "default_category": "Network activity", + "to_ids": 1 + }, + "domain|ip": { + "default_category": "Network activity", + "to_ids": 1 + }, + "email-src": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "email-dst": { + "default_category": "Network activity", + "to_ids": 1 + }, + "email-subject": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "email-attachment": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "url": { + "default_category": "External analysis", + "to_ids": 1 + }, + "http-method": { + "default_category": "Network activity", + "to_ids": 0 + }, + "user-agent": { + "default_category": "Network activity", + "to_ids": 0 + }, + "regkey": { + "default_category": "Persistence mechanism", + "to_ids": 1 + }, + "regkey|value": { + "default_category": "Persistence mechanism", + "to_ids": 1 + }, + "AS": { + "default_category": "Network activity", + "to_ids": 0 + }, + "snort": { + "default_category": "Network activity", + "to_ids": 1 + }, + "pattern-in-file": { + "default_category": "Payload installation", + "to_ids": 1 + }, + "pattern-in-traffic": { + "default_category": "Network activity", + "to_ids": 1 + }, + "pattern-in-memory": { + "default_category": "Payload installation", + "to_ids": 1 + }, + "yara": { + "default_category": "Payload installation", + "to_ids": 1 + }, + "vulnerability": { + "default_category": "External analysis", + "to_ids": 0 + }, + "attachment": { + "default_category": "External analysis", + "to_ids": 0 + }, + "malware-sample": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "link": { + "default_category": "External analysis", + "to_ids": 0 + }, + "comment": { + "default_category": "Other", + "to_ids": 0 + }, + "text": { + "default_category": "Other", + "to_ids": 0 + }, + "other": { + "default_category": "Other", + "to_ids": 0 + }, + "named pipe": { + "default_category": "Artifacts dropped", + "to_ids": 0 + }, + "mutex": { + "default_category": "Artifacts dropped", + "to_ids": 1 + }, + "target-user": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "target-email": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "target-machine": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "target-org": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "target-location": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "target-external": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "btc": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "iban": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "bic": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "bank-account-nr": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "aba-rtn": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "bin": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "cc-number": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "prtn": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "threat-actor": { + "default_category": "Attribution", + "to_ids": 0 + }, + "campaign-name": { + "default_category": "Attribution", + "to_ids": 0 + }, + "campaign-id": { + "default_category": "Attribution", + "to_ids": 0 + }, + "malware-type": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "uri": { + "default_category": "Network activity", + "to_ids": 1 + }, + "authentihash": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "ssdeep": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "imphash": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "pehash": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "sha224": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "sha384": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "sha512": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "sha512/224": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "sha512/256": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "tlsh": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|authentihash": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|ssdeep": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|imphash": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|pehash": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha224": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha384": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha512": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha512/224": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha512/256": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|tlsh": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "windows-scheduled-task": { + "default_category": "Artifacts dropped", + "to_ids": 0 + }, + "windows-service-name": { + "default_category": "Artifacts dropped", + "to_ids": 0 + }, + "windows-service-displayname": { + "default_category": "Artifacts dropped", + "to_ids": 0 + }, + "whois-registrant-email": { + "default_category": "Attribution", + "to_ids": 0 + }, + "whois-registrant-phone": { + "default_category": "Attribution", + "to_ids": 0 + }, + "whois-registrant-name": { + "default_category": "Attribution", + "to_ids": 0 + }, + "whois-registrar": { + "default_category": "Attribution", + "to_ids": 0 + }, + "whois-creation-date": { + "default_category": "Attribution", + "to_ids": 0 + }, + "x509-fingerprint-sha1": { + "default_category": "Network activity", + "to_ids": 1 + } + }, + "types": [ + "md5", + "sha1", + "sha256", + "filename", + "pdb", + "filename|md5", + "filename|sha1", + "filename|sha256", + "ip-src", + "ip-dst", + "hostname", + "domain", + "domain|ip", + "email-src", + "email-dst", + "email-subject", + "email-attachment", + "url", + "http-method", + "user-agent", + "regkey", + "regkey|value", + "AS", + "snort", + "pattern-in-file", + "pattern-in-traffic", + "pattern-in-memory", + "yara", + "vulnerability", + "attachment", + "malware-sample", + "link", + "comment", + "text", + "other", + "named pipe", + "mutex", + "target-user", + "target-email", + "target-machine", + "target-org", + "target-location", + "target-external", + "btc", + "iban", + "bic", + "bank-account-nr", + "aba-rtn", + "bin", + "cc-number", + "prtn", + "threat-actor", + "campaign-name", + "campaign-id", + "malware-type", + "uri", + "authentihash", + "ssdeep", + "imphash", + "pehash", + "sha224", + "sha384", + "sha512", + "sha512/224", + "sha512/256", + "tlsh", + "filename|authentihash", + "filename|ssdeep", + "filename|imphash", + "filename|pehash", + "filename|sha224", + "filename|sha384", + "filename|sha512", + "filename|sha512/224", + "filename|sha512/256", + "filename|tlsh", + "windows-scheduled-task", + "windows-service-name", + "windows-service-displayname", + "whois-registrant-email", + "whois-registrant-phone", + "whois-registrant-name", + "whois-registrar", + "whois-creation-date", + "x509-fingerprint-sha1" + ], + "categories": [ + "Internal reference", + "Targeting data", + "Antivirus detection", + "Payload delivery", + "Artifacts dropped", + "Payload installation", + "Persistence mechanism", + "Network activity", + "Payload type", + "Attribution", + "External analysis", + "Financial fraud", + "Other" + ], + "category_type_mappings": { + "Internal reference": [ + "text", + "link", + "comment", + "other" + ], + "Targeting data": [ + "target-user", + "target-email", + "target-machine", + "target-org", + "target-location", + "target-external", + "comment" + ], + "Antivirus detection": [ + "link", + "comment", + "text", + "attachment", + "other" + ], + "Payload delivery": [ + "md5", + "sha1", + "sha224", + "sha256", + "sha384", + "sha512", + "sha512/224", + "sha512/256", + "ssdeep", + "imphash", + "authentihash", + "pehash", + "tlsh", + "filename", + "filename|md5", + "filename|sha1", + "filename|sha224", + "filename|sha256", + "filename|sha384", + "filename|sha512", + "filename|sha512/224", + "filename|sha512/256", + "filename|authentihash", + "filename|ssdeep", + "filename|tlsh", + "filename|imphash", + "filename|pehash", + "ip-src", + "ip-dst", + "hostname", + "domain", + "email-src", + "email-dst", + "email-subject", + "email-attachment", + "url", + "user-agent", + "AS", + "pattern-in-file", + "pattern-in-traffic", + "yara", + "attachment", + "malware-sample", + "link", + "malware-type", + "comment", + "text", + "vulnerability", + "x509-fingerprint-sha1", + "other" + ], + "Artifacts dropped": [ + "md5", + "sha1", + "sha224", + "sha256", + "sha384", + "sha512", + "sha512/224", + "sha512/256", + "ssdeep", + "imphash", + "authentihash", + "filename", + "filename|md5", + "filename|sha1", + "filename|sha224", + "filename|sha256", + "filename|sha384", + "filename|sha512", + "filename|sha512/224", + "filename|sha512/256", + "filename|authentihash", + "filename|ssdeep", + "filename|tlsh", + "filename|imphash", + "filename|pehash", + "regkey", + "regkey|value", + "pattern-in-file", + "pattern-in-memory", + "pdb", + "yara", + "attachment", + "malware-sample", + "named pipe", + "mutex", + "windows-scheduled-task", + "windows-service-name", + "windows-service-displayname", + "comment", + "text", + "x509-fingerprint-sha1", + "other" + ], + "Payload installation": [ + "md5", + "sha1", + "sha224", + "sha256", + "sha384", + "sha512", + "sha512/224", + "sha512/256", + "ssdeep", + "imphash", + "authentihash", + "pehash", + "tlsh", + "filename", + "filename|md5", + "filename|sha1", + "filename|sha224", + "filename|sha256", + "filename|sha384", + "filename|sha512", + "filename|sha512/224", + "filename|sha512/256", + "filename|authentihash", + "filename|ssdeep", + "filename|tlsh", + "filename|imphash", + "filename|pehash", + "pattern-in-file", + "pattern-in-traffic", + "pattern-in-memory", + "yara", + "vulnerability", + "attachment", + "malware-sample", + "malware-type", + "comment", + "text", + "x509-fingerprint-sha1", + "other" + ], + "Persistence mechanism": [ + "filename", + "regkey", + "regkey|value", + "comment", + "text", + "other" + ], + "Network activity": [ + "ip-src", + "ip-dst", + "hostname", + "domain", + "domain|ip", + "email-dst", + "url", + "uri", + "user-agent", + "http-method", + "AS", + "snort", + "pattern-in-file", + "pattern-in-traffic", + "attachment", + "comment", + "text", + "x509-fingerprint-sha1", + "other" + ], + "Payload type": [ + "comment", + "text", + "other" + ], + "Attribution": [ + "threat-actor", + "campaign-name", + "campaign-id", + "whois-registrant-phone", + "whois-registrant-email", + "whois-registrant-name", + "whois-registrar", + "whois-creation-date", + "comment", + "text", + "x509-fingerprint-sha1", + "other" + ], + "External analysis": [ + "md5", + "sha1", + "sha256", + "filename", + "filename|md5", + "filename|sha1", + "filename|sha256", + "ip-src", + "ip-dst", + "hostname", + "domain", + "domain|ip", + "url", + "user-agent", + "regkey", + "regkey|value", + "AS", + "snort", + "pattern-in-file", + "pattern-in-traffic", + "pattern-in-memory", + "vulnerability", + "attachment", + "malware-sample", + "link", + "comment", + "text", + "x509-fingerprint-sha1", + "other" + ], + "Financial fraud": [ + "btc", + "iban", + "bic", + "bank-account-nr", + "aba-rtn", + "bin", + "cc-number", + "prtn", + "comment", + "text", + "other" + ], + "Other": [ + "comment", + "text", + "other" + ] + } + } +} From 8be38693b7a466d54f8c20c6c02d1bb690471708 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 10 Oct 2016 13:42:06 +0200 Subject: [PATCH 134/223] Fix test --- tests/test_offline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_offline.py b/tests/test_offline.py index b08d5c3..b4854af 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -121,7 +121,7 @@ class TestOffline(unittest.TestCase): def test_eventObject(self, m): self.initURI(m) pymisp = PyMISP(self.domain, self.key) - misp_event = MISPEvent(pymisp.describe_types['result']) + misp_event = MISPEvent(pymisp.describe_types) misp_event.load(open('tests/57c4445b-c548-4654-af0b-4be3950d210f.json', 'r').read()) json.dumps(misp_event, cls=EncodeUpdate) json.dumps(misp_event, cls=EncodeFull) From 268598fe231d1802145ceac5442540df5b9ecd24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 11 Oct 2016 11:22:31 +0200 Subject: [PATCH 135/223] Add forgotten variable --- pymisp/api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pymisp/api.py b/pymisp/api.py index 0d01298..b1bad95 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -92,6 +92,7 @@ class PyMISP(object): self.ssl = ssl self.proxies = proxies self.cert = cert + self.ressources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') if out_type != 'json': raise PyMISPError('The only output type supported by PyMISP is JSON. If you still rely on XML, use PyMISP v2.4.49') self.debug = debug From 3df35826c5b2c483b0d0cfb357071eecb71daa2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Tue, 11 Oct 2016 11:26:57 +0200 Subject: [PATCH 136/223] fix flatten --- pymisp/api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pymisp/api.py b/pymisp/api.py index b1bad95..409aa35 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -159,6 +159,8 @@ class PyMISP(object): messages.append('Error in {}: {}'.format(where, msg)) else: for e in errors: + if not e: + continue if isinstance(e, str): messages.append(e) continue From ed2be78f4e99a7d8900f47b12f85e1c92d09e52e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 11 Oct 2016 17:45:38 +0200 Subject: [PATCH 137/223] Avoid validation issue if attribute_count is none --- pymisp/mispevent.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 678b084..87c259f 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -208,16 +208,23 @@ class MISPEvent(object): def load(self, json_event): self.new = False self.dump_full = True + if isinstance(json_event, basestring) and os.path.exists(json_event): + # NOTE: is it a good idea? (possible security issue if an untrusted user call this method) + json_event = open(json_event, 'r') + if hasattr(json_event, 'read'): + # python2 and python3 compatible to find if we have a file + json_event = json_event.read() if isinstance(json_event, basestring): - loaded = json.loads(json_event) - if loaded.get('response'): - event = loaded.get('response')[0] - else: - event = loaded - if not event: - raise PyMISPError('Invalid event') + json_event = json.loads(json_event) + if json_event.get('response'): + event = json_event.get('response')[0] else: event = json_event + if not event: + raise PyMISPError('Invalid event') + # Invalid event created by MISP up to 2.4.52 (attribute_count is none instead of '0') + if event.get('Event') and event.get('Event').get('attribute_count') is None: + event['Event']['attribute_count'] = '0' jsonschema.validate(event, self.json_schema_lax) e = event.get('Event') self._reinitialize_event() From bc5df41179a6db6ffd90a9ddb5176def0a3b11e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Wed, 12 Oct 2016 12:33:42 +0200 Subject: [PATCH 138/223] fix situational-awareness examples --- .../attribute_treemap.py | 13 ++-- examples/situational-awareness/tag_search.py | 44 +++++++------ examples/situational-awareness/tags_count.py | 29 +++++---- .../situational-awareness/tags_to_graphs.py | 65 ++++++++++--------- examples/situational-awareness/tools.py | 4 +- 5 files changed, 84 insertions(+), 71 deletions(-) diff --git a/examples/situational-awareness/attribute_treemap.py b/examples/situational-awareness/attribute_treemap.py index 0536590..33ab6b5 100755 --- a/examples/situational-awareness/attribute_treemap.py +++ b/examples/situational-awareness/attribute_treemap.py @@ -21,8 +21,11 @@ if __name__ == '__main__': else: result = misp.download_last(args.argument) - events = tools.eventsListBuildFromArray(result) - attributes = tools.attributesListBuild(events) - temp = tools.getNbAttributePerEventCategoryType(attributes) - temp = temp.groupby(level=['category', 'type']).sum() - tools.createTreemap(temp, 'Attributes Distribution', 'attribute_treemap.svg', 'attribute_table.html') + if 'response' in result: + events = tools.eventsListBuildFromArray(result) + attributes = tools.attributesListBuild(events) + temp = tools.getNbAttributePerEventCategoryType(attributes) + temp = temp.groupby(level=['category', 'type']).sum() + tools.createTreemap(temp, 'Attributes Distribution', 'attribute_treemap.svg', 'attribute_table.html') + else: + print ('There is no event answering the research criteria') diff --git a/examples/situational-awareness/tag_search.py b/examples/situational-awareness/tag_search.py index 490d0ff..e2250c2 100644 --- a/examples/situational-awareness/tag_search.py +++ b/examples/situational-awareness/tag_search.py @@ -41,25 +41,29 @@ if __name__ == '__main__': else: args.enddate = tools.setEnddate(tools.toDatetime(args.enddate)) - events = tools.selectInRange(tools.eventsListBuildFromArray(result), begin=args.begindate, end=args.enddate) - totalPeriodEvents = tools.getNbitems(events) - tags = tools.tagsListBuild(events) - result = tools.isTagIn(tags, args.tag) - totalPeriodTags = len(result) + if 'response' in result: + events = tools.selectInRange(tools.eventsListBuildFromArray(result), begin=args.begindate, end=args.enddate) + totalPeriodEvents = tools.getNbitems(events) + tags = tools.tagsListBuild(events) + result = tools.isTagIn(tags, args.tag) + totalPeriodTags = len(result) - text = 'Studied pediod: from ' - if args.begindate is None: - text = text + '1970-01-01' - else: - text = text + str(args.begindate.date()) - text = text + ' to ' - if args.enddate is None: - text = text + str(datetime.now().date()) - else: - text = text + str(args.enddate.date()) + text = 'Studied pediod: from ' + if args.begindate is None: + text = text + '1970-01-01' + else: + text = text + str(args.begindate.date()) + text = text + ' to ' + if args.enddate is None: + text = text + str(datetime.now().date()) + else: + text = text + str(args.enddate.date()) + + print('\n========================================================') + print(text) + print('During the studied pediod, ' + str(totalPeriodTags) + ' events out of ' + str(totalPeriodEvents) + ' contains at least one tag with ' + args.tag + '.') + if totalPeriodEvents != 0: + print('It represents {}% of the events in this period.'.format(round(100 * totalPeriodTags / totalPeriodEvents, 3))) + else: + print ('There is no event answering the research criteria') - print('\n========================================================') - print(text) - print('During the studied pediod, ' + str(totalPeriodTags) + ' events out of ' + str(totalPeriodEvents) + ' contains at least one tag with ' + args.tag + '.') - if totalPeriodEvents != 0: - print('It represents {}% of the events in this period.'.format(round(100 * totalPeriodTags / totalPeriodEvents, 3))) diff --git a/examples/situational-awareness/tags_count.py b/examples/situational-awareness/tags_count.py index f925574..e0b49bf 100644 --- a/examples/situational-awareness/tags_count.py +++ b/examples/situational-awareness/tags_count.py @@ -40,20 +40,23 @@ if __name__ == '__main__': else: args.enddate = tools.setEnddate(tools.toDatetime(args.enddate)) - events = tools.selectInRange(tools.eventsListBuildFromArray(result), begin=args.begindate, end=args.enddate) - tags = tools.tagsListBuild(events) - result = tools.getNbOccurenceTags(tags) + if 'response' in result: + events = tools.selectInRange(tools.eventsListBuildFromArray(result), begin=args.begindate, end=args.enddate) + tags = tools.tagsListBuild(events) + result = tools.getNbOccurenceTags(tags) + else: + result = 'There is no event during the studied period' - text = 'Studied pediod: from ' - if args.begindate is None: - text = text + '1970-01-01' - else: - text = text + str(args.begindate.date()) - text = text + ' to ' - if args.enddate is None: - text = text + str(datetime.now().date()) - else: - text = text + str(args.enddate.date()) + text = 'Studied pediod: from ' + if args.begindate is None: + text = text + '1970-01-01' + else: + text = text + str(args.begindate.date()) + text = text + ' to ' + if args.enddate is None: + text = text + str(datetime.now().date()) + else: + text = text + str(args.enddate.date()) print('\n========================================================') print(text) diff --git a/examples/situational-awareness/tags_to_graphs.py b/examples/situational-awareness/tags_to_graphs.py index 7280165..3923b51 100644 --- a/examples/situational-awareness/tags_to_graphs.py +++ b/examples/situational-awareness/tags_to_graphs.py @@ -50,42 +50,45 @@ if __name__ == '__main__': title = 'Tags repartition over the last 7 days' result = misp.download_last(last) - events = tools.eventsListBuildFromArray(result) - result = [] - dates = [] - enddate = tools.getToday() - colourDict = {} - faketag = False + if 'response' in result: + events = tools.eventsListBuildFromArray(result) + result = [] + dates = [] + enddate = tools.getToday() + colourDict = {} + faketag = False - for i in range(split): - begindate = tools.getNDaysBefore(enddate, size) - dates.append(str(enddate.date())) - eventstemp = tools.selectInRange(events, begin=begindate, end=enddate) - if eventstemp is not None: - tags = tools.tagsListBuild(eventstemp) - if tags is not None: - tools.createDictTagsColour(colourDict, tags) - result.append(tools.getNbOccurenceTags(tags)) + for i in range(split): + begindate = tools.getNDaysBefore(enddate, size) + dates.append(str(enddate.date())) + eventstemp = tools.selectInRange(events, begin=begindate, end=enddate) + if eventstemp is not None: + tags = tools.tagsListBuild(eventstemp) + if tags is not None: + tools.createDictTagsColour(colourDict, tags) + result.append(tools.getNbOccurenceTags(tags)) + else: + result.append(tools.createFakeEmptyTagsSeries()) + faketag = True else: result.append(tools.createFakeEmptyTagsSeries()) faketag = True - else: - result.append(tools.createFakeEmptyTagsSeries()) - faketag = True - enddate = begindate + enddate = begindate - result = formattingDataframe(result, dates, 0) - if faketag: - result = tools.removeFaketagRow(result) + result = formattingDataframe(result, dates, 0) + if faketag: + result = tools.removeFaketagRow(result) - taxonomies, emptyOther = tools.getTaxonomies(tools.getCopyDataframe(result)) + taxonomies, emptyOther = tools.getTaxonomies(tools.getCopyDataframe(result)) + tools.tagsToLineChart(tools.getCopyDataframe(result), title, dates, colourDict) + tools.tagstrendToLineChart(tools.getCopyDataframe(result), title, dates, split, colourDict) + tools.tagsToTaxoLineChart(tools.getCopyDataframe(result), title, dates, colourDict, taxonomies, emptyOther) + tools.tagstrendToTaxoLineChart(tools.getCopyDataframe(result), title, dates, split, colourDict, taxonomies, emptyOther) + if args.order is None: + args.order = 3 + tools.tagsToPolyChart(tools.getCopyDataframe(result), split, colourDict, taxonomies, emptyOther, args.order) + tools.createVisualisation(taxonomies) - tools.tagsToLineChart(tools.getCopyDataframe(result), title, dates, colourDict) - tools.tagstrendToLineChart(tools.getCopyDataframe(result), title, dates, split, colourDict) - tools.tagsToTaxoLineChart(tools.getCopyDataframe(result), title, dates, colourDict, taxonomies, emptyOther) - tools.tagstrendToTaxoLineChart(tools.getCopyDataframe(result), title, dates, split, colourDict, taxonomies, emptyOther) - if args.order is None: - args.order = 3 - tools.tagsToPolyChart(tools.getCopyDataframe(result), split, colourDict, taxonomies, emptyOther, args.order) - tools.createVisualisation(taxonomies) + else: + print('There is no event during the studied period') diff --git a/examples/situational-awareness/tools.py b/examples/situational-awareness/tools.py index d4b9bea..694eb2b 100644 --- a/examples/situational-awareness/tools.py +++ b/examples/situational-awareness/tools.py @@ -437,7 +437,7 @@ def tagsToPolyChart(dataframe, split, colourDict, taxonomies, emptyOther, order) pylab.title('Polynomial Fit with Matplotlib: ' + taxonomy) pylab.legend(loc='center left', bbox_to_anchor=(1, 0.5)) ax = plt.gca() - ax.set_facecolor((0.898, 0.898, 0.898)) + # ax.set_facecolor((0.898, 0.898, 0.898)) box = ax.get_position() ax.set_position([box.x0 - 0.01, box.y0, box.width * 0.78, box.height]) fig = plt.gcf() @@ -473,7 +473,7 @@ def tagsToPolyChart(dataframe, split, colourDict, taxonomies, emptyOther, order) pylab.title('Polynomial Fit with Matplotlib: other') pylab.legend(loc='center left', bbox_to_anchor=(1, 0.5)) ax = plt.gca() - ax.set_facecolor((0.898, 0.898, 0.898)) + #cax.set_facecolor((0.898, 0.898, 0.898)) box = ax.get_position() ax.set_position([box.x0 - 0.01, box.y0, box.width * 0.78, box.height]) fig = plt.gcf() From 9cc55341f0e1f98f17e1a460b1c0fbace4bbe3e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Wed, 12 Oct 2016 15:40:49 +0200 Subject: [PATCH 139/223] fix date formatting in mispevent.py + some PEP8 cleaning --- examples/events/tools.py | 36 +++++++++++++++++++++++------------- pymisp/mispevent.py | 2 +- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/examples/events/tools.py b/examples/events/tools.py index 322aace..9d0e3f5 100644 --- a/examples/events/tools.py +++ b/examples/events/tools.py @@ -5,60 +5,70 @@ import random from random import randint import string + def randomStringGenerator(size, chars=string.ascii_lowercase + string.digits): return ''.join(random.choice(chars) for _ in range(size)) + def randomIpGenerator(): return str(randint(0, 255)) + '.' + str(randint(0, 255)) + '.' + str(randint(0, 255)) + '.' + str(randint(0, 255)) -def floodtxt(misp, event, maxlength = 255): + +def floodtxt(misp, event, maxlength=255): text = randomStringGenerator(randint(1, maxlength)) textfunctions = [misp.add_internal_comment, misp.add_internal_text, misp.add_internal_other, misp.add_email_subject, misp.add_mutex, misp.add_filename] - textfunctions[randint(0,5)](event, text) + textfunctions[randint(0, 5)](event, text) + def floodip(misp, event): ip = randomIpGenerator() ipfunctions = [misp.add_ipsrc, misp.add_ipdst] - ipfunctions[randint(0,1)](event, ip) + ipfunctions[randint(0, 1)](event, ip) -def flooddomain(misp, event, maxlength = 25): + +def flooddomain(misp, event, maxlength=25): a = randomStringGenerator(randint(1, maxlength)) b = randomStringGenerator(randint(2, 3), chars=string.ascii_lowercase) domain = a + '.' + b domainfunctions = [misp.add_hostname, misp.add_domain] - domainfunctions[randint(0,1)](event, domain) + domainfunctions[randint(0, 1)](event, domain) -def flooddomainip(misp, event, maxlength = 25): + +def flooddomainip(misp, event, maxlength=25): a = randomStringGenerator(randint(1, maxlength)) b = randomStringGenerator(randint(2, 3), chars=string.ascii_lowercase) domain = a + '.' + b ip = randomIpGenerator() misp.add_domain_ip(event, domain, ip) -def floodemail(misp, event, maxlength = 25): + +def floodemail(misp, event, maxlength=25): a = randomStringGenerator(randint(1, maxlength)) b = randomStringGenerator(randint(1, maxlength)) c = randomStringGenerator(randint(2, 3), chars=string.ascii_lowercase) - email = a + '@' + b + '.'+ c + email = a + '@' + b + '.' + c emailfunctions = [misp.add_email_src, misp.add_email_dst] - emailfunctions[randint(0,1)](event, email) + emailfunctions[randint(0, 1)](event, email) + def floodattachment(misp, eventid, distribution, to_ids, category, comment, info, analysis, threat_level_id): - filename = randomStringGenerator(randint(1,128)) + filename = randomStringGenerator(randint(1, 128)) misp.upload_sample(filename, 'dummy', eventid, distribution, to_ids, category, comment, info, analysis, threat_level_id) + def create_dummy_event(misp): event = misp.new_event(0, 4, 0, 'dummy event') flooddomainip(misp, event) - floodattachment(misp, event['Event']['id'], event['Event']['id'], event['Event']['distribution'], False, 'Payload delivery', '', event['Event']['info'], event['Event']['analysis'], event['Event']['threat_level_id']) + floodattachment(misp, event['Event']['id'], event['Event']['distribution'], False, 'Payload delivery', '', event['Event']['info'], event['Event']['analysis'], event['Event']['threat_level_id']) + def create_massive_dummy_events(misp, nbattribute): event = misp.new_event(0, 4, 0, 'massive dummy event') eventid = event['Event']['id'] functions = [floodtxt, floodip, flooddomain, flooddomainip, floodemail, floodattachment] for i in range(nbattribute): - choice = randint(0,5) + choice = randint(0, 5) if choice == 5: floodattachment(misp, eventid, event['Event']['distribution'], False, 'Payload delivery', '', event['Event']['info'], event['Event']['analysis'], event['Event']['threat_level_id']) else: - functions[choice](misp,event) + functions[choice](misp, event) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 87c259f..12e22cd 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -254,7 +254,7 @@ class MISPEvent(object): self.publish() if kwargs.get('date'): if isinstance(kwargs['date'], basestring): - self.date = parse(kwargs['date']) + self.date = parse(kwargs['date']).date() elif isinstance(kwargs['date'], datetime.datetime): self.date = kwargs['date'].date() elif isinstance(kwargs['date'], datetime.date): From b1e6765bb372d863f9f105486f566722b32cc9d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Thu, 13 Oct 2016 10:11:18 +0200 Subject: [PATCH 140/223] fix indentation --- examples/situational-awareness/tags_count.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/examples/situational-awareness/tags_count.py b/examples/situational-awareness/tags_count.py index e0b49bf..5ab2899 100644 --- a/examples/situational-awareness/tags_count.py +++ b/examples/situational-awareness/tags_count.py @@ -47,16 +47,16 @@ if __name__ == '__main__': else: result = 'There is no event during the studied period' - text = 'Studied pediod: from ' - if args.begindate is None: - text = text + '1970-01-01' - else: - text = text + str(args.begindate.date()) + text = 'Studied pediod: from ' + if args.begindate is None: + text = text + '1970-01-01' + else: + text = text + str(args.begindate.date()) text = text + ' to ' - if args.enddate is None: - text = text + str(datetime.now().date()) - else: - text = text + str(args.enddate.date()) + if args.enddate is None: + text = text + str(datetime.now().date()) + else: + text = text + str(args.enddate.date()) print('\n========================================================') print(text) From 22956fbd81a860627e7b704b8540308e7af74abd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Thu, 13 Oct 2016 11:48:17 +0200 Subject: [PATCH 141/223] add metadata flag to search --- pymisp/api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pymisp/api.py b/pymisp/api.py index 409aa35..804156e 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -828,7 +828,7 @@ class PyMISP(object): def search(self, values=None, not_values=None, type_attribute=None, category=None, org=None, tags=None, not_tags=None, date_from=None, - date_to=None, last=None, controller='events'): + date_to=None, last=None, metadata=None, controller='events'): """ Search via the Rest API @@ -869,6 +869,8 @@ class PyMISP(object): query['to'] = date_to if last is not None: query['last'] = last + if metadata is not None: + query['metadata'] = metadata session = self.__prepare_session() return self.__query(session, 'restSearch/download', query, controller) From 13dbb96111b973f7a27e0a9fff76585ff91b6ea6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Thu, 13 Oct 2016 13:39:44 +0200 Subject: [PATCH 142/223] Use only metadata in situational awareness tags functions --- examples/situational-awareness/tag_search.py | 2 +- examples/situational-awareness/tags_count.py | 2 +- examples/situational-awareness/tags_to_graphs.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/situational-awareness/tag_search.py b/examples/situational-awareness/tag_search.py index e2250c2..20d422d 100644 --- a/examples/situational-awareness/tag_search.py +++ b/examples/situational-awareness/tag_search.py @@ -27,7 +27,7 @@ if __name__ == '__main__': if args.days is None: args.days = 7 - result = misp.download_last('{}d'.format(args.days)) + result = misp.search(last='{}d'.format(args.days), metadata=True) tools.checkDateConsistancy(args.begindate, args.enddate, tools.getLastdate(args.days)) diff --git a/examples/situational-awareness/tags_count.py b/examples/situational-awareness/tags_count.py index 5ab2899..c58ca5b 100644 --- a/examples/situational-awareness/tags_count.py +++ b/examples/situational-awareness/tags_count.py @@ -26,7 +26,7 @@ if __name__ == '__main__': if args.days is None: args.days = 7 - result = misp.download_last('{}d'.format(args.days)) + result = misp.search(last='{}d'.format(args.days), metadata=True) tools.checkDateConsistancy(args.begindate, args.enddate, tools.getLastdate(args.days)) diff --git a/examples/situational-awareness/tags_to_graphs.py b/examples/situational-awareness/tags_to_graphs.py index 3923b51..0bad486 100644 --- a/examples/situational-awareness/tags_to_graphs.py +++ b/examples/situational-awareness/tags_to_graphs.py @@ -5,7 +5,7 @@ from pymisp import PyMISP from keys import misp_url, misp_key, misp_verifycert import argparse import tools - +import timing def formattingDataframe(dataframe, dates, NanValue): dataframe.reverse() @@ -49,7 +49,7 @@ if __name__ == '__main__': last = '7d' title = 'Tags repartition over the last 7 days' - result = misp.download_last(last) + result = misp.search(last=last, metadata=True) if 'response' in result: events = tools.eventsListBuildFromArray(result) result = [] From 0d0ca1e9b530c2b6c03a153a900af546408ae059 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Thu, 13 Oct 2016 13:43:57 +0200 Subject: [PATCH 143/223] Update comment --- pymisp/api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pymisp/api.py b/pymisp/api.py index 804156e..9841303 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -842,6 +842,7 @@ class PyMISP(object): :param date_from: First date :param date_to: Last date :param last: Last updated events (for example 5d or 12h or 30m) + :param metadata: return onlymetadata if True """ val = self.__prepare_rest_search(values, not_values).replace('/', '|') From 30cd45e94ee7b6f5ac591ddf423dce7f103f8d7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Thu, 13 Oct 2016 15:28:18 +0200 Subject: [PATCH 144/223] remove test import --- examples/situational-awareness/tags_to_graphs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/situational-awareness/tags_to_graphs.py b/examples/situational-awareness/tags_to_graphs.py index 0bad486..76464a4 100644 --- a/examples/situational-awareness/tags_to_graphs.py +++ b/examples/situational-awareness/tags_to_graphs.py @@ -5,7 +5,7 @@ from pymisp import PyMISP from keys import misp_url, misp_key, misp_verifycert import argparse import tools -import timing + def formattingDataframe(dataframe, dates, NanValue): dataframe.reverse() From 3455567d45e326051875512bb3264ea2375648eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 19 Oct 2016 18:29:20 +0200 Subject: [PATCH 145/223] Fix schemas --- pymisp/data/schema-lax.json | 2 +- pymisp/data/schema.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pymisp/data/schema-lax.json b/pymisp/data/schema-lax.json index f21ac64..09bc780 100644 --- a/pymisp/data/schema-lax.json +++ b/pymisp/data/schema-lax.json @@ -225,7 +225,7 @@ }, "Event": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event", - "type": "array", + "type": "object", "items": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0", "type": "object", diff --git a/pymisp/data/schema.json b/pymisp/data/schema.json index d7cb964..85f3e3d 100644 --- a/pymisp/data/schema.json +++ b/pymisp/data/schema.json @@ -225,7 +225,7 @@ }, "Event": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event", - "type": "array", + "type": "object", "items": { "id": "https://www.github.com/MISP/MISP/format/2.4/schema.json/Event/RelatedEvent/0/Event/0", "type": "object", From 49433cdc3a22684e6066a22b59d5eab4080bc36c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 19 Oct 2016 18:46:47 +0200 Subject: [PATCH 146/223] Print a warning in case python2 is used. --- pymisp/api.py | 3 +++ pymisp/mispevent.py | 1 + 2 files changed, 4 insertions(+) diff --git a/pymisp/api.py b/pymisp/api.py index 9841303..0fd0349 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -8,8 +8,10 @@ import datetime import os import base64 import re +import warnings try: + warnings.warn("You're using python 2, it is strongly recommended to use python >=3.3") from urllib.parse import urljoin except ImportError: from urlparse import urljoin @@ -29,6 +31,7 @@ from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate # Least dirty way to support python 2 and 3 try: + warnings.warn("You're using python 2, it is strongly recommended to use python >=3.3") basestring except NameError: basestring = str diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 12e22cd..7d94361 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -20,6 +20,7 @@ from .exceptions import PyMISPError, NewEventError, NewAttributeError # Least dirty way to support python 2 and 3 try: + warnings.warn("You're using python 2, it is strongly recommended to use python >=3.3") basestring except NameError: basestring = str From 0a63285efb2348ae0814e6599f43df4f038bd2b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 19 Oct 2016 19:00:09 +0200 Subject: [PATCH 147/223] Fix Python2 support --- pymisp/mispevent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 7d94361..b436aab 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -254,7 +254,7 @@ class MISPEvent(object): if kwargs.get('published') is not None: self.publish() if kwargs.get('date'): - if isinstance(kwargs['date'], basestring): + if isinstance(kwargs['date'], basestring) or isinstance(kwargs['date'], unicode): self.date = parse(kwargs['date']).date() elif isinstance(kwargs['date'], datetime.datetime): self.date = kwargs['date'].date() From ec508191ca260acde4b1e88b38359a86c433a809 Mon Sep 17 00:00:00 2001 From: Tom Date: Thu, 20 Oct 2016 09:49:06 +0100 Subject: [PATCH 148/223] Including Network-Other option for API --- pymisp/api.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pymisp/api.py b/pymisp/api.py index c44577b..8844d90 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -606,6 +606,11 @@ class PyMISP(object): attributes.append(self._prepare_full_attribute(category, 'snort', snort, to_ids, comment, distribution)) return self._send_attributes(event, attributes, proposal) + def add_net_other(self, event, netother, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): + attributes = [] + attributes.append(self._prepare_full_attribute(category, 'other', netother, to_ids, comment, distribution)) + return self._send_attributes(event, attributes, proposal) + # ##### Email attributes ##### def add_email_src(self, event, email, to_ids=True, comment=None, distribution=None, proposal=False): From f8f117f1a46d8189044612dc6b752c952d6dab1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 21 Oct 2016 13:42:22 +0200 Subject: [PATCH 149/223] Cleanup --- pymisp/api.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index edd2a4e..16cacd1 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -76,9 +76,6 @@ class PyMISP(object): :param cert: Client certificate, as described there: http://docs.python-requests.org/en/master/user/advanced/#ssl-cert-verification """ -# TODO: interactive script to create a MISP event from scratch -# TODO: a parser to verify the validity of an event - # So it can may be accessed from the misp object. distributions = distributions threat_level = threat_level @@ -1026,6 +1023,7 @@ class PyMISP(object): return {'version': '{}.{}.{}'.format(master_version['major'], master_version['minor'], master_version['hotfix'])} else: return {'error': 'Impossible to retrieve the version of the master branch.'} + # ############## Export Attributes in text #################################### def get_all_attributes_txt(self, type_attr): From c8aeab782362c0086ca12e2e3f5d08846077b3b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 21 Oct 2016 15:42:38 +0200 Subject: [PATCH 150/223] Bump to v2.4.53 --- pymisp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 7a17886..7cbe0ee 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.4.51.1' +__version__ = '2.4.53' from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey from .api import PyMISP From 0de3f7459bf3557671cf7d39b8da80743f20c4a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Sat, 22 Oct 2016 14:52:17 +0200 Subject: [PATCH 151/223] add example add_named_argument.py --- examples/add_named_attribute.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 examples/add_named_attribute.py diff --git a/examples/add_named_attribute.py b/examples/add_named_attribute.py new file mode 100644 index 0000000..43bb5db --- /dev/null +++ b/examples/add_named_attribute.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pymisp import PyMISP +from keys import misp_url, misp_key +import argparse + +# For python2 & 3 compat, a bit dirty, but it seems to be the least bad one +try: + input = raw_input +except NameError: + pass + + +def init(url, key): + return PyMISP(url, key, True, 'json', debug=True) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Create an event on MISP.') + parser.add_argument("-e", "--event", type=int, help="The id of the event to update.") + parser.add_argument("-t", "--type", help="The type of the added attribute") + parser.add_argument("-v", "--value", help="The value of the attribute") + args = parser.parse_args() + + misp = init(misp_url, misp_key) + + event = misp.get_event(args.event) + event = misp.add_named_attribute(event, args.type, args.value) + print(event) From 5ccb4db3d3dc91a122628111ea01300b9063249d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 25 Oct 2016 11:33:51 -0400 Subject: [PATCH 152/223] Allow to load a MISP event without attributes --- pymisp/data/schema-lax.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pymisp/data/schema-lax.json b/pymisp/data/schema-lax.json index 09bc780..dd4af2f 100644 --- a/pymisp/data/schema-lax.json +++ b/pymisp/data/schema-lax.json @@ -311,8 +311,7 @@ } }, "required": [ - "info", - "Attribute" + "info" ] } }, From 2c2bfe2354fcce3d9067d15cb6d57e845a05a2e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 25 Oct 2016 11:36:49 -0400 Subject: [PATCH 153/223] Properly warn if the user is using python2 --- pymisp/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 16cacd1..7566086 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -11,10 +11,10 @@ import re import warnings try: - warnings.warn("You're using python 2, it is strongly recommended to use python >=3.3") from urllib.parse import urljoin except ImportError: from urlparse import urljoin + warnings.warn("You're using python 2, it is strongly recommended to use python >=3.3") from io import BytesIO import zipfile @@ -31,8 +31,8 @@ from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate # Least dirty way to support python 2 and 3 try: - warnings.warn("You're using python 2, it is strongly recommended to use python >=3.3") basestring + warnings.warn("You're using python 2, it is strongly recommended to use python >=3.3") except NameError: basestring = str From abd836babb81a325562afbffcb3f9de23fa7075d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 25 Oct 2016 17:28:55 -0400 Subject: [PATCH 154/223] Add simple script to push MISP events into Neo4j --- examples/make_neo4j.py | 32 ++++++++++++++++++++++ pymisp/__init__.py | 1 + pymisp/tools/__init__.py | 0 pymisp/tools/neo4j.py | 59 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 92 insertions(+) create mode 100755 examples/make_neo4j.py create mode 100644 pymisp/tools/__init__.py create mode 100644 pymisp/tools/neo4j.py diff --git a/examples/make_neo4j.py b/examples/make_neo4j.py new file mode 100755 index 0000000..247a866 --- /dev/null +++ b/examples/make_neo4j.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pymisp import PyMISP +from pymisp import Neo4j +from pymisp import MISPEvent +from keys import misp_url, misp_key +import argparse + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Get all the events matching a value.') + parser.add_argument("-s", "--search", required=True, help="String to search.") + parser.add_argument("--host", default='localhost:7474', help="Host where neo4j is running.") + parser.add_argument("-u", "--user", default='neo4j', help="User on neo4j.") + parser.add_argument("-p", "--password", default='neo4j', help="Password on neo4j.") + args = parser.parse_args() + + neo4j = Neo4j(args.host, args.user, args.password) + neo4j.del_all() + misp = PyMISP(misp_url, misp_key) + result = misp.search_all(args.search) + for json_event in result['response']: + if not json_event['Event']: + print(json_event) + continue + print('Importing', json_event['Event']['info'], json_event['Event']['id']) + try: + misp_event = MISPEvent() + misp_event.load(json_event) + neo4j.import_event(misp_event) + except: + print('broken') diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 7cbe0ee..e8553e2 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -3,3 +3,4 @@ __version__ = '2.4.53' from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey from .api import PyMISP from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull +from .tools.neo4j import Neo4j diff --git a/pymisp/tools/__init__.py b/pymisp/tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pymisp/tools/neo4j.py b/pymisp/tools/neo4j.py new file mode 100644 index 0000000..cb49df1 --- /dev/null +++ b/pymisp/tools/neo4j.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import glob +import os +from pymisp import MISPEvent + +try: + from py2neo import authenticate, Graph, Node, Relationship + has_py2neo = True +except ImportError: + has_py2neo = False + + +class Neo4j(): + + def __init__(self, host='localhost:7474', username='neo4j', password='neo4j'): + if not has_py2neo: + raise Exception('py2neo is required, please install: pip install py2neo') + authenticate(host, username, password) + self.graph = Graph() + + def load_events_directory(self, directory): + self.events = [] + for path in glob.glob(os.path.join(directory, '*.json')): + e = MISPEvent() + e.load(path) + self.import_event(e) + + def del_all(self): + self.graph.delete_all() + + def import_event(self, event): + tx = self.graph.begin() + event_node = Node('Event', uuid=event.uuid) + event_node['name'] = event.info + # event_node['distribution'] = event.distribution + # event_node['threat_level_id'] = event.threat_level_id + # event_node['analysis'] = event.analysis + # event_node['published'] = event.published + # event_node['date'] = event.date.isoformat() + tx.create(event_node) + for a in event.attributes: + attr_node = Node('Attribute', a.type, uuid=a.uuid) + attr_node['category'] = a.category + attr_node['name'] = a.value + # attr_node['to_ids'] = a.to_ids + # attr_node['comment'] = a.comment + # attr_node['distribution'] = a.distribution + tx.create(attr_node) + member_rel = Relationship(event_node, "is member", attr_node) + tx.create(member_rel) + val = Node('Value', name=a.value) + ev = Relationship(event_node, "has", val) + av = Relationship(attr_node, "is", val) + s = val | ev | av + tx.merge(s) + tx.graph.push(s) + tx.commit() From 2907fd18d775bfd23070eb1db1bb99fd700652ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 27 Oct 2016 15:58:08 -0400 Subject: [PATCH 155/223] Cleanup neo4j support --- examples/make_neo4j.py | 4 +++- pymisp/tools/neo4j.py | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/make_neo4j.py b/examples/make_neo4j.py index 247a866..26b568e 100755 --- a/examples/make_neo4j.py +++ b/examples/make_neo4j.py @@ -13,10 +13,12 @@ if __name__ == '__main__': parser.add_argument("--host", default='localhost:7474', help="Host where neo4j is running.") parser.add_argument("-u", "--user", default='neo4j', help="User on neo4j.") parser.add_argument("-p", "--password", default='neo4j', help="Password on neo4j.") + parser.add_argument("-d", "--deleteall", action="store_true", default=False, help="Delete all nodes from the database") args = parser.parse_args() neo4j = Neo4j(args.host, args.user, args.password) - neo4j.del_all() + if args.deleteall: + neo4j.del_all() misp = PyMISP(misp_url, misp_key) result = misp.search_all(args.search) for json_event in result['response']: diff --git a/pymisp/tools/neo4j.py b/pymisp/tools/neo4j.py index cb49df1..af5ba2c 100644 --- a/pymisp/tools/neo4j.py +++ b/pymisp/tools/neo4j.py @@ -32,8 +32,7 @@ class Neo4j(): def import_event(self, event): tx = self.graph.begin() - event_node = Node('Event', uuid=event.uuid) - event_node['name'] = event.info + event_node = Node('Event', uuid=event.uuid, name=event.info) # event_node['distribution'] = event.distribution # event_node['threat_level_id'] = event.threat_level_id # event_node['analysis'] = event.analysis From d48f2481763a4bf8eb7c075fdeedf0e438708c6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 27 Oct 2016 15:58:58 -0400 Subject: [PATCH 156/223] Fix Python2 - Python3 support --- pymisp/mispevent.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index b436aab..bb3a024 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -20,10 +20,11 @@ from .exceptions import PyMISPError, NewEventError, NewAttributeError # Least dirty way to support python 2 and 3 try: - warnings.warn("You're using python 2, it is strongly recommended to use python >=3.3") basestring + warnings.warn("You're using python 2, it is strongly recommended to use python >=3.3") except NameError: basestring = str + unicode = str class MISPAttribute(object): From ac2e801d978e07111c1e4a3f922e1fe337b37c1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 27 Oct 2016 16:25:17 -0400 Subject: [PATCH 157/223] Add helper tool to load STIX objects. --- pymisp/tools/stix.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 pymisp/tools/stix.py diff --git a/pymisp/tools/stix.py b/pymisp/tools/stix.py new file mode 100644 index 0000000..b964295 --- /dev/null +++ b/pymisp/tools/stix.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +try: + from misp_stix_converter.converters.buildMISPAttribute import buildEvent, open_stix + from misp_stix_converter.converters.convert import MISPtoSTIX + has_misp_stix_converter = True +except ImportError: + has_misp_stix_converter = False + + +def load_stix(stix, distribution=3, threat_level_id=2, analysis=0): + '''Returns a MISPEvent object from a STIX package''' + if not has_misp_stix_converter: + raise Exception('You need to install misp_stix_converter from https://github.com/MISP/MISP-STIX-Converter') + stix = open_stix(stix) + return buildEvent(stix, distribution=distribution, + threat_level_id=threat_level_id, analysis=analysis) + + +def make_stix_package(misp_event, to_json=False, to_xml=False): + '''Returns a STIXPackage from a MISPEvent. + + Optionally can return the package in json or xml. + + ''' + if not has_misp_stix_converter: + raise Exception('You need to install misp_stix_converter from https://github.com/MISP/MISP-STIX-Converter') + package = MISPtoSTIX(misp_event) + if to_json: + return package.to_json() + elif to_xml: + return package.to_xml() + else: + return package From 857cd40ea20d9c79282170ad6185dc53f5c9a9c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 27 Oct 2016 16:29:56 -0400 Subject: [PATCH 158/223] Update import --- pymisp/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index e8553e2..dac6257 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -4,3 +4,4 @@ from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDe from .api import PyMISP from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull from .tools.neo4j import Neo4j +from .tools import stix From 81e3ce37afd617f18cdd5acc13d8dcebd53989f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 27 Oct 2016 17:04:23 -0400 Subject: [PATCH 159/223] Fix forgotten import --- pymisp/mispevent.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index bb3a024..5fa82c8 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -6,6 +6,7 @@ import time import json from json import JSONEncoder import os +import warnings try: from dateutil.parser import parse except ImportError: @@ -21,6 +22,7 @@ from .exceptions import PyMISPError, NewEventError, NewAttributeError # Least dirty way to support python 2 and 3 try: basestring + unicode warnings.warn("You're using python 2, it is strongly recommended to use python >=3.3") except NameError: basestring = str From bee1630e98320f8e00a2eeec9dbfc5e457d1b901 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 28 Oct 2016 14:13:57 -0400 Subject: [PATCH 160/223] Add query example --- examples/make_neo4j.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/examples/make_neo4j.py b/examples/make_neo4j.py index 26b568e..6393813 100755 --- a/examples/make_neo4j.py +++ b/examples/make_neo4j.py @@ -7,6 +7,17 @@ from pymisp import MISPEvent from keys import misp_url, misp_key import argparse +""" +Sample Neo4J query: + + +MATCH ()-[r:has]->(n) +WITH n, count(r) as rel_cnt +WHERE rel_cnt > 5 +MATCH (m)-[r:has]->(n) +RETURN m, n LIMIT 200; +""" + if __name__ == '__main__': parser = argparse.ArgumentParser(description='Get all the events matching a value.') parser.add_argument("-s", "--search", required=True, help="String to search.") From 6c5289d495d44809ed0943dfd29247f680a82a3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sat, 29 Oct 2016 15:27:48 -0400 Subject: [PATCH 161/223] Initial version of the OpenIOC loader --- pymisp/mispevent.py | 22 +++++--- pymisp/tools/openioc.py | 118 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 8 deletions(-) create mode 100644 pymisp/tools/openioc.py diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 5fa82c8..e0ecb73 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -234,6 +234,19 @@ class MISPEvent(object): self._reinitialize_event() self.set_all_values(**e) + def set_date(self, date, ignore_invalid=False): + if isinstance(date, basestring) or isinstance(date, unicode): + self.date = parse(date).date() + elif isinstance(date, datetime.datetime): + self.date = date.date() + elif isinstance(date, datetime.date): + self.date = date + else: + if ignore_invalid: + self.date = datetime.date.today() + else: + raise NewEventError('Invalid format for the date: {} - {}'.format(date, type(date))) + def set_all_values(self, **kwargs): # Required value if kwargs.get('info'): @@ -257,14 +270,7 @@ class MISPEvent(object): if kwargs.get('published') is not None: self.publish() if kwargs.get('date'): - if isinstance(kwargs['date'], basestring) or isinstance(kwargs['date'], unicode): - self.date = parse(kwargs['date']).date() - elif isinstance(kwargs['date'], datetime.datetime): - self.date = kwargs['date'].date() - elif isinstance(kwargs['date'], datetime.date): - self.date = kwargs['date'] - else: - raise NewEventError('Invalid format for the date: {} - {}'.format(kwargs['date'], type(kwargs['date']))) + self.set_date(kwargs['date']) if kwargs.get('Attribute'): for a in kwargs['Attribute']: attribute = MISPAttribute(self.describe_types) diff --git a/pymisp/tools/openioc.py b/pymisp/tools/openioc.py new file mode 100644 index 0000000..7c89b95 --- /dev/null +++ b/pymisp/tools/openioc.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pymisp import MISPEvent +try: + from bs4 import BeautifulSoup + has_bs4 = True +except ImportError: + has_bs4 = False + + +iocMispMapping = { + 'DriverItem/DriverName': {'category': 'Artifacts dropped', 'type': 'other', 'comment': 'DriverName.'}, + + 'DnsEntryItem/Host': {'type': 'domain'}, + + 'Email/To': {'type': 'target-email'}, + 'Email/Date': {'type': 'comment', 'comment': 'EmailDate.'}, + 'Email/Body': {'type': 'email-subject'}, + 'Email/From': {'type': 'email-dst'}, + 'Email/Subject': {'type': 'email-subject'}, + 'Email/Attachment/Name': {'type': 'email-attachment'}, + + 'FileItem/Md5sum': {'type': 'md5'}, + 'FileItem/Sha1sum': {'type': 'sha1'}, + 'FileItem/Sha256sum': {'type': 'sha256'}, + 'FileItem/FileName': {'type': 'filename'}, + 'FileItem/FullPath': {'type': 'filename'}, + 'FileItem/FilePath': {'type': 'filename'}, + + 'Network/URI': {'type': 'uri'}, + 'Network/DNS': {'type': 'domain'}, + 'Network/String': {'type': 'ip-dst'}, + 'Network/UserAgent': {'type': 'user-agent'}, + + 'PortItem/localIP': {'type': 'ip-dst'}, + + 'ProcessItem/name': {'type': 'pattern-in-memory', 'comment': 'ProcessName.'}, + 'ProcessItem/path': {'type': 'pattern-in-memory', 'comment': 'ProcessPath.'}, + 'ProcessItem/Mutex': {'type': 'mutex'}, + 'ProcessItem/Pipe/Name': {'type': 'named pipe'}, + 'ProcessItem/Mutex/Name': {'type': 'mutex', 'comment': 'MutexName.'}, + + # Is it the regkey value? + # 'RegistryItem/Text': {'type': 'regkey', 'RegistryText. '}, + 'RegistryItem/Path': {'type': 'regkey'}, + + 'ServiceItem/name': {'type': 'windows-service-name'}, + 'ServiceItem/type': {'type': 'pattern-in-memory', 'comment': 'ServiceType. '}, + + 'Snort/Snort': {'type': 'snort'}, +} + + +def extract_field(report, field_name): + data = report.find(field_name.lower()) + if data and hasattr(data, 'text'): + return data.text + return None + + +def load_openioc(openioc): + if not has_bs4: + raise Exception('You need to install BeautifulSoup: pip install bs4') + misp_event = MISPEvent() + with open(openioc, "r") as ioc_file: + iocreport = BeautifulSoup(ioc_file, "lxml") + # Set event fields + info = extract_field(iocreport, 'short_description') + if info: + misp_event.info = info + date = extract_field(iocreport, 'authored_date') + if date: + misp_event.set_date(date) + # Set special attributes + description = extract_field(iocreport, 'description') + if description: + misp_event.add_attribute('comment', description) + author = extract_field(iocreport, 'authored_by') + if author: + misp_event.add_attribute('comment', author) + misp_event = set_all_attributes(iocreport, misp_event) + return misp_event + + +def get_mapping(openioc_type): + t = openioc_type.lower() + for k, v in iocMispMapping.items(): + if k.lower() == t: + return v + return False + + +def set_all_attributes(openioc, misp_event): + for item in openioc.find_all("indicatoritem"): + print(item) + attribute_values = {'comment': ''} + if item.find('context'): + mapping = get_mapping(item.find('context')['search']) + if mapping: + attribute_values.update(mapping) + else: + # Unknown mapping, ignoring + # print(item.find('context')) + continue + else: + continue + value = extract_field(openioc, 'Content') + if value: + attribute_values['value'] = value + else: + # No value, ignoring + continue + comment = extract_field(openioc, 'Comment') + if comment: + attribute_values["comment"] = '{} {}'.format(attribute_values["comment"], comment) + misp_event.add_attribute(**attribute_values) + return misp_event From 0b462404de246579992a37892161cd30107d65ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Thu, 3 Nov 2016 11:23:48 +0100 Subject: [PATCH 162/223] add user management and examples --- examples/add_user.py | 28 ++++++++++++++++++++++++ examples/delete_user.py | 25 +++++++++++++++++++++ examples/edit_user.py | 29 ++++++++++++++++++++++++ examples/sighting.py | 4 +--- examples/user_sample.json | 6 +++++ examples/users_list.py | 24 ++++++++++++++++++++ pymisp/api.py | 46 +++++++++++++++++++++++++++++++++++++++ 7 files changed, 159 insertions(+), 3 deletions(-) create mode 100755 examples/add_user.py create mode 100755 examples/delete_user.py create mode 100755 examples/edit_user.py create mode 100644 examples/user_sample.json create mode 100644 examples/users_list.py diff --git a/examples/add_user.py b/examples/add_user.py new file mode 100755 index 0000000..6236f09 --- /dev/null +++ b/examples/add_user.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pymisp import PyMISP +from keys import misp_url, misp_key +import argparse + +# For python2 & 3 compat, a bit dirty, but it seems to be the least bad one +try: + input = raw_input +except NameError: + pass + + +def init(url, key): + return PyMISP(url, key, True, 'json') + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Add the user described in the given json. If no file is provided, returns a json listing all the fields used to describe a user.') + parser.add_argument("-f", "--json_file", help="The name of the json file describing the user you want to create.") + args = parser.parse_args() + + misp = init(misp_url, misp_key) + + if args.json_file is None: + print (misp.get_add_user_fields_list()) + else: + print(misp.add_user(args.json_file)) diff --git a/examples/delete_user.py b/examples/delete_user.py new file mode 100755 index 0000000..b6aaf7d --- /dev/null +++ b/examples/delete_user.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pymisp import PyMISP +from keys import misp_url, misp_key +import argparse + +# For python2 & 3 compat, a bit dirty, but it seems to be the least bad one +try: + input = raw_input +except NameError: + pass + + +def init(url, key): + return PyMISP(url, key, True, 'json') + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Delete the user with the given id. Keep in mind that disabling users (by setting the disabled flag via an edit) is always prefered to keep user associations to events intact.') + parser.add_argument("-i", "--user_id", help="The id of the user you want to delete.") + args = parser.parse_args() + + misp = init(misp_url, misp_key) + + print(misp.delete_user(args.user_id)) diff --git a/examples/edit_user.py b/examples/edit_user.py new file mode 100755 index 0000000..f7b3d98 --- /dev/null +++ b/examples/edit_user.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pymisp import PyMISP +from keys import misp_url, misp_key +import argparse + +# For python2 & 3 compat, a bit dirty, but it seems to be the least bad one +try: + input = raw_input +except NameError: + pass + + +def init(url, key): + return PyMISP(url, key, True, 'json') + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Edit the user designed by the user_id. If no file is provided, returns a json listing all the fields used to describe a user.') + parser.add_argument("-i", "--user_id", required=True, help="The name of the json file describing the user you want to modify.") + parser.add_argument("-f", "--json_file", help="The name of the json file describing your modifications.") + args = parser.parse_args() + + misp = init(misp_url, misp_key) + + if args.json_file is None: + print (misp.get_edit_user_fields_list(args.user_id)) + else: + print(misp.edit_user(args.json_file, args.user_id)) diff --git a/examples/sighting.py b/examples/sighting.py index 433c2c2..10bd72d 100755 --- a/examples/sighting.py +++ b/examples/sighting.py @@ -17,11 +17,9 @@ def init(url, key): if __name__ == '__main__': parser = argparse.ArgumentParser(description='Add sighting.') - parser.add_argument("-f", "--json_file", help="The name of the json file describing the attribute you want to add sighting to.") + parser.add_argument("-f", "--json_file", required=True, help="The name of the json file describing the attribute you want to add sighting to.") args = parser.parse_args() misp = init(misp_url, misp_key) misp.sighting_per_json(args.json_file) - - diff --git a/examples/user_sample.json b/examples/user_sample.json new file mode 100644 index 0000000..626b519 --- /dev/null +++ b/examples/user_sample.json @@ -0,0 +1,6 @@ +{ + "email":"maaiil@domain.lu", + "org_id":1, + "role_id":1, + "autoalert":1 +} diff --git a/examples/users_list.py b/examples/users_list.py new file mode 100644 index 0000000..78620ee --- /dev/null +++ b/examples/users_list.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pymisp import PyMISP +from keys import misp_url, misp_key +import argparse + +# For python2 & 3 compat, a bit dirty, but it seems to be the least bad one +try: + input = raw_input +except NameError: + pass + + +def init(url, key): + return PyMISP(url, key, True, 'json') + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Get a list of the sharing groups from the MISP instance.') + + misp = init(misp_url, misp_key) + + users_list = misp.get_users_list() + print (users_list) diff --git a/pymisp/api.py b/pymisp/api.py index 16cacd1..327b128 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1094,3 +1094,49 @@ class PyMISP(object): url = urljoin(self.root_url, 'sharing_groups/index.json') response = session.get(url) return self._check_response(response)['response'][0] + + # ############## Users ################## + + def get_users_list(self): + session = self.__prepare_session() + url = urljoin(self.root_url, 'admin/users') + response = session.get(url) + return self._check_response(response)['response'] + + def get_user(self, user_id): + session = self.__prepare_session() + url = urljoin(self.root_url, 'admin/users/view/{}'.format(user_id)) + response = session.get(url) + return self._check_response(response) + + def add_user(self, json_file): + session = self.__prepare_session() + jdata = json.load(open(json_file)) + url = urljoin(self.root_url, 'admin/users/add/') + response = session.post(url, data=json.dumps(jdata)) + return self._check_response(response) + + def get_add_user_fields_list(self): + session = self.__prepare_session() + url = urljoin(self.root_url, 'admin/users/add/') + response = session.get(url) + return self._check_response(response) + + def edit_user(self, json_file, user_id): + session = self.__prepare_session() + jdata = json.load(open(json_file)) + url = urljoin(self.root_url, 'admin/users/edit/{}'.format(user_id)) + response = session.post(url, data=json.dumps(jdata)) + return self._check_response(response) + + def get_edit_user_fields_list(self, user_id): + session = self.__prepare_session() + url = urljoin(self.root_url, 'admin/users/edit/{}'.format(user_id)) + response = session.get(url) + return self._check_response(response) + + def delete_user(self, user_id): + session = self.__prepare_session() + url = urljoin(self.root_url, 'admin/users/delete/{}'.format(user_id)) + response = session.post(url) + return self._check_response(response) From 9f0737c34a965938fc3608bb080a8b83e263a4e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 3 Nov 2016 16:01:48 -0400 Subject: [PATCH 163/223] Add some mapping to openioc, add python version in the user agent --- pymisp/api.py | 3 ++- pymisp/tools/openioc.py | 12 +++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 7566086..113c691 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -3,6 +3,7 @@ """Python API using the REST interface of MISP""" +import sys import json import datetime import os @@ -136,7 +137,7 @@ class PyMISP(object): {'Authorization': self.key, 'Accept': 'application/{}'.format(output), 'content-type': 'application/{}'.format(output), - 'User-Agent': 'PyMISP {}'.format(__version__)}) + 'User-Agent': 'PyMISP {} - Python {}.{}.{}'.format(__version__, *sys.version_info)}) return session def flatten_error_messages(self, response): diff --git a/pymisp/tools/openioc.py b/pymisp/tools/openioc.py index 7c89b95..e6be33d 100644 --- a/pymisp/tools/openioc.py +++ b/pymisp/tools/openioc.py @@ -23,14 +23,17 @@ iocMispMapping = { 'FileItem/Md5sum': {'type': 'md5'}, 'FileItem/Sha1sum': {'type': 'sha1'}, + 'TaskItem/Sha1sum': {'type': 'sha1'}, 'FileItem/Sha256sum': {'type': 'sha256'}, 'FileItem/FileName': {'type': 'filename'}, 'FileItem/FullPath': {'type': 'filename'}, 'FileItem/FilePath': {'type': 'filename'}, + 'DriverItem/DriverName': {'type': 'filename'}, 'Network/URI': {'type': 'uri'}, 'Network/DNS': {'type': 'domain'}, 'Network/String': {'type': 'ip-dst'}, + 'RouteEntryItem/Destination': {'type': 'ip-dst'}, 'Network/UserAgent': {'type': 'user-agent'}, 'PortItem/localIP': {'type': 'ip-dst'}, @@ -41,8 +44,16 @@ iocMispMapping = { 'ProcessItem/Pipe/Name': {'type': 'named pipe'}, 'ProcessItem/Mutex/Name': {'type': 'mutex', 'comment': 'MutexName.'}, + 'CookieHistoryItem/HostName': {'type': 'hostname'}, + 'FormHistoryItem/HostName': {'type': 'hostname'}, + 'SystemInfoItem/HostName': {'type': 'hostname'}, + 'UrlHistoryItem/HostName': {'type': 'hostname'}, + 'DnsEntryItem/RecordName': {'type': 'hostname'}, + 'DnsEntryItem/Host': {'type': 'hostname'}, + # Is it the regkey value? # 'RegistryItem/Text': {'type': 'regkey', 'RegistryText. '}, + 'RegistryItem/KeyPath': {'type': 'regkey'}, 'RegistryItem/Path': {'type': 'regkey'}, 'ServiceItem/name': {'type': 'windows-service-name'}, @@ -93,7 +104,6 @@ def get_mapping(openioc_type): def set_all_attributes(openioc, misp_event): for item in openioc.find_all("indicatoritem"): - print(item) attribute_values = {'comment': ''} if item.find('context'): mapping = get_mapping(item.find('context')['search']) From bbf919878712a8f334b79ed21c28b950c86ce269 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Fri, 4 Nov 2016 09:31:31 +0100 Subject: [PATCH 164/223] Moving Neo4j into graphdb --- examples/make_neo4j.py | 45 ------------------------------------------ 1 file changed, 45 deletions(-) delete mode 100755 examples/make_neo4j.py diff --git a/examples/make_neo4j.py b/examples/make_neo4j.py deleted file mode 100755 index 6393813..0000000 --- a/examples/make_neo4j.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from pymisp import PyMISP -from pymisp import Neo4j -from pymisp import MISPEvent -from keys import misp_url, misp_key -import argparse - -""" -Sample Neo4J query: - - -MATCH ()-[r:has]->(n) -WITH n, count(r) as rel_cnt -WHERE rel_cnt > 5 -MATCH (m)-[r:has]->(n) -RETURN m, n LIMIT 200; -""" - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Get all the events matching a value.') - parser.add_argument("-s", "--search", required=True, help="String to search.") - parser.add_argument("--host", default='localhost:7474', help="Host where neo4j is running.") - parser.add_argument("-u", "--user", default='neo4j', help="User on neo4j.") - parser.add_argument("-p", "--password", default='neo4j', help="Password on neo4j.") - parser.add_argument("-d", "--deleteall", action="store_true", default=False, help="Delete all nodes from the database") - args = parser.parse_args() - - neo4j = Neo4j(args.host, args.user, args.password) - if args.deleteall: - neo4j.del_all() - misp = PyMISP(misp_url, misp_key) - result = misp.search_all(args.search) - for json_event in result['response']: - if not json_event['Event']: - print(json_event) - continue - print('Importing', json_event['Event']['info'], json_event['Event']['id']) - try: - misp_event = MISPEvent() - misp_event.load(json_event) - neo4j.import_event(misp_event) - except: - print('broken') From 55b4a0725bcd091c959df1b4b9b29d50980a3d5d Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Fri, 4 Nov 2016 09:31:52 +0100 Subject: [PATCH 165/223] Neo4j stuff moved into graphdb directory --- examples/graphdb/make_neo4j.py | 45 ++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100755 examples/graphdb/make_neo4j.py diff --git a/examples/graphdb/make_neo4j.py b/examples/graphdb/make_neo4j.py new file mode 100755 index 0000000..6393813 --- /dev/null +++ b/examples/graphdb/make_neo4j.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pymisp import PyMISP +from pymisp import Neo4j +from pymisp import MISPEvent +from keys import misp_url, misp_key +import argparse + +""" +Sample Neo4J query: + + +MATCH ()-[r:has]->(n) +WITH n, count(r) as rel_cnt +WHERE rel_cnt > 5 +MATCH (m)-[r:has]->(n) +RETURN m, n LIMIT 200; +""" + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Get all the events matching a value.') + parser.add_argument("-s", "--search", required=True, help="String to search.") + parser.add_argument("--host", default='localhost:7474', help="Host where neo4j is running.") + parser.add_argument("-u", "--user", default='neo4j', help="User on neo4j.") + parser.add_argument("-p", "--password", default='neo4j', help="Password on neo4j.") + parser.add_argument("-d", "--deleteall", action="store_true", default=False, help="Delete all nodes from the database") + args = parser.parse_args() + + neo4j = Neo4j(args.host, args.user, args.password) + if args.deleteall: + neo4j.del_all() + misp = PyMISP(misp_url, misp_key) + result = misp.search_all(args.search) + for json_event in result['response']: + if not json_event['Event']: + print(json_event) + continue + print('Importing', json_event['Event']['info'], json_event['Event']['id']) + try: + misp_event = MISPEvent() + misp_event.load(json_event) + neo4j.import_event(misp_event) + except: + print('broken') From a11e26f80bc937800e1cb057e5b996dde1db2494 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Fri, 4 Nov 2016 11:58:21 +0100 Subject: [PATCH 166/223] Improvements in the user api --- examples/add_user.py | 11 ++-- examples/add_user_json.py | 28 ++++++++++ examples/edit_user.py | 9 ++-- examples/edit_user_json.py | 29 +++++++++++ pymisp/api.py | 102 ++++++++++++++++++++++++++++++++++++- 5 files changed, 165 insertions(+), 14 deletions(-) create mode 100755 examples/add_user_json.py create mode 100755 examples/edit_user_json.py diff --git a/examples/add_user.py b/examples/add_user.py index 6236f09..fbec04e 100755 --- a/examples/add_user.py +++ b/examples/add_user.py @@ -16,13 +16,12 @@ def init(url, key): return PyMISP(url, key, True, 'json') if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Add the user described in the given json. If no file is provided, returns a json listing all the fields used to describe a user.') - parser.add_argument("-f", "--json_file", help="The name of the json file describing the user you want to create.") + parser = argparse.ArgumentParser(description='Add a new user by setting the mandory fields.') + parser.add_argument("-e", "--email", required=True, help="Email linked to the account.") + parser.add_argument("-o", "--org_id", required=True, help="Organisation linked to the user.") + parser.add_argument("-r", "--role_id", required=True, help="Role linked to the user.") args = parser.parse_args() misp = init(misp_url, misp_key) - if args.json_file is None: - print (misp.get_add_user_fields_list()) - else: - print(misp.add_user(args.json_file)) + print (misp.add_user(args.email, args.org_id, args.role_id)) diff --git a/examples/add_user_json.py b/examples/add_user_json.py new file mode 100755 index 0000000..6f79cc1 --- /dev/null +++ b/examples/add_user_json.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pymisp import PyMISP +from keys import misp_url, misp_key +import argparse + +# For python2 & 3 compat, a bit dirty, but it seems to be the least bad one +try: + input = raw_input +except NameError: + pass + + +def init(url, key): + return PyMISP(url, key, True, 'json') + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Add the user described in the given json. If no file is provided, returns a json listing all the fields used to describe a user.') + parser.add_argument("-f", "--json_file", help="The name of the json file describing the user you want to create.") + args = parser.parse_args() + + misp = init(misp_url, misp_key) + + if args.json_file is None: + print (misp.get_add_user_fields_list()) + else: + print(misp.add_user_json(args.json_file)) diff --git a/examples/edit_user.py b/examples/edit_user.py index f7b3d98..6d16ea9 100755 --- a/examples/edit_user.py +++ b/examples/edit_user.py @@ -16,14 +16,11 @@ def init(url, key): return PyMISP(url, key, True, 'json') if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Edit the user designed by the user_id. If no file is provided, returns a json listing all the fields used to describe a user.') + parser = argparse.ArgumentParser(description='Edit the email of the user designed by the user_id.') parser.add_argument("-i", "--user_id", required=True, help="The name of the json file describing the user you want to modify.") - parser.add_argument("-f", "--json_file", help="The name of the json file describing your modifications.") + parser.add_argument("-e", "--email", help="Email linked to the account.") args = parser.parse_args() misp = init(misp_url, misp_key) - if args.json_file is None: - print (misp.get_edit_user_fields_list(args.user_id)) - else: - print(misp.edit_user(args.json_file, args.user_id)) + print(misp.edit_user(args.user_id, email=args.email)) diff --git a/examples/edit_user_json.py b/examples/edit_user_json.py new file mode 100755 index 0000000..7c5deb8 --- /dev/null +++ b/examples/edit_user_json.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pymisp import PyMISP +from keys import misp_url, misp_key +import argparse + +# For python2 & 3 compat, a bit dirty, but it seems to be the least bad one +try: + input = raw_input +except NameError: + pass + + +def init(url, key): + return PyMISP(url, key, True, 'json') + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Edit the user designed by the user_id. If no file is provided, returns a json listing all the fields used to describe a user.') + parser.add_argument("-i", "--user_id", required=True, help="The name of the json file describing the user you want to modify.") + parser.add_argument("-f", "--json_file", help="The name of the json file describing your modifications.") + args = parser.parse_args() + + misp = init(misp_url, misp_key) + + if args.json_file is None: + print (misp.get_edit_user_fields_list(args.user_id)) + else: + print(misp.edit_user_json(args.json_file, args.user_id)) diff --git a/pymisp/api.py b/pymisp/api.py index 701f19b..3b7eb04 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1109,7 +1109,58 @@ class PyMISP(object): response = session.get(url) return self._check_response(response) - def add_user(self, json_file): + def add_user_json(self, json_file): + session = self.__prepare_session() + jdata = json.load(open(json_file)) + url = urljoin(self.root_url, 'admin/users/add/') + response = session.post(url, data=json.dumps(jdata)) + return self._check_response(response) + + def add_user(self, email, org_id, role_id, password=None, + external_auth_required=None, external_auth_key=None, + enable_password=None, nids_sid=None, server_id=None, + gpgkey=None, certif_public=None, autoalert=None, + contactalert=None, disabled=None, change_pw=None, + termsaccepted=None, newsread=None): + new_user = {} + new_user['email'] = email + new_user['org_id'] = org_id + new_user['role_id'] = role_id + if password is not None: + new_user['password'] = password + if external_auth_required is not None: + new_user['external_auth_required'] = external_auth_required + if external_auth_key is not None: + new_user['external_auth_key'] = external_auth_key + if enable_password is not None: + new_user['enable_password'] = enable_password + if nids_sid is not None: + new_user['nids_sid'] = nids_sid + if server_id is not None: + new_user['server_id'] = server_id + if gpgkey is not None: + new_user['gpgkey'] = gpgkey + if certif_public is not None: + new_user['certif_public'] = certif_public + if autoalert is not None: + new_user['autoalert'] = autoalert + if contactalert is not None: + new_user['contactalert'] = contactalert + if disabled is not None: + new_user['disabled'] = disabled + if change_pw is not None: + new_user['change_pw'] = change_pw + if termsaccepted is not None: + new_user['termsaccepted'] = termsaccepted + if newsread is not None: + new_user['newsread'] = newsread + + session = self.__prepare_session() + url = urljoin(self.root_url, 'admin/users/add/') + response = session.post(url, data=json.dumps(new_user)) + return self._check_response(response) + + def add_user_json(self, json_file): session = self.__prepare_session() jdata = json.load(open(json_file)) url = urljoin(self.root_url, 'admin/users/add/') @@ -1122,7 +1173,54 @@ class PyMISP(object): response = session.get(url) return self._check_response(response) - def edit_user(self, json_file, user_id): + def edit_user(self, user_id, email=None, org_id=None, role_id=None, + password=None, external_auth_required=None, + external_auth_key=None, enable_password=None, nids_sid=None, + server_id=None, gpgkey=None, certif_public=None, + autoalert=None, contactalert=None, disabled=None, + change_pw=None, termsaccepted=None, newsread=None): + edit_user = {} + if email is not None: + edit_user['email'] = email + if org_id is not None: + edit_user['org_id'] = org_id + if role_id is not None: + edit_user['role_id'] = role_id + if password is not None: + edit_user['password'] = password + if external_auth_required is not None: + edit_user['external_auth_required'] = external_auth_required + if external_auth_key is not None: + edit_user['external_auth_key'] = external_auth_key + if enable_password is not None: + edit_user['enable_password'] = enable_password + if nids_sid is not None: + edit_user['nids_sid'] = nids_sid + if server_id is not None: + edit_user['server_id'] = server_id + if gpgkey is not None: + edit_user['gpgkey'] = gpgkey + if certif_public is not None: + edit_user['certif_public'] = certif_public + if autoalert is not None: + edit_user['autoalert'] = autoalert + if contactalert is not None: + edit_user['contactalert'] = contactalert + if disabled is not None: + edit_user['disabled'] = disabled + if change_pw is not None: + edit_user['change_pw'] = change_pw + if termsaccepted is not None: + edit_user['termsaccepted'] = termsaccepted + if newsread is not None: + edit_user['newsread'] = newsread + + session = self.__prepare_session() + url = urljoin(self.root_url, 'admin/users/edit/{}'.format(user_id)) + response = session.post(url, data=json.dumps(edit_user)) + return self._check_response(response) + + def edit_user_json(self, json_file, user_id): session = self.__prepare_session() jdata = json.load(open(json_file)) url = urljoin(self.root_url, 'admin/users/edit/{}'.format(user_id)) From a72b4d77798645a1f232dbc0f77d2bf20af1ab78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 4 Nov 2016 14:32:29 +0100 Subject: [PATCH 167/223] Fix package installation --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7a465ac..7ddac16 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ setup( maintainer='Raphaël Vinot', url='https://github.com/MISP/PyMISP', description='Python API for MISP.', - packages=['pymisp'], + packages=['pymisp', 'pymisp.tools'], classifiers=[ 'License :: OSI Approved :: BSD License', 'Development Status :: 5 - Production/Stable', From 477f81d3abdbaf28f4fc0d19bc5cf42c8f0c3c0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 10 Nov 2016 11:41:13 +0100 Subject: [PATCH 168/223] Fix openioc loader, update mapping --- pymisp/tools/openioc.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/pymisp/tools/openioc.py b/pymisp/tools/openioc.py index e6be33d..808f260 100644 --- a/pymisp/tools/openioc.py +++ b/pymisp/tools/openioc.py @@ -16,15 +16,23 @@ iocMispMapping = { 'Email/To': {'type': 'target-email'}, 'Email/Date': {'type': 'comment', 'comment': 'EmailDate.'}, - 'Email/Body': {'type': 'email-subject'}, + # 'Email/Body': {'type': 'email-subject'}, 'Email/From': {'type': 'email-dst'}, 'Email/Subject': {'type': 'email-subject'}, 'Email/Attachment/Name': {'type': 'email-attachment'}, 'FileItem/Md5sum': {'type': 'md5'}, 'FileItem/Sha1sum': {'type': 'sha1'}, - 'TaskItem/Sha1sum': {'type': 'sha1'}, 'FileItem/Sha256sum': {'type': 'sha256'}, + + 'ServiceItem/serviceDLLmd5sum': {'type': 'md5', 'category': 'Payload installation'}, + 'ServiceItem/serviceDLLsha1sum': {'type': 'sha1', 'category': 'Payload installation'}, + 'ServiceItem/serviceDLLsha256sum': {'type': 'sha256', 'category': 'Payload installation'}, + + 'TaskItem/md5sum': {'type': 'md5'}, + 'TaskItem/sha1sum': {'type': 'sha1'}, + 'TaskItem/Sha256sum': {'type': 'sha256'}, + 'FileItem/FileName': {'type': 'filename'}, 'FileItem/FullPath': {'type': 'filename'}, 'FileItem/FilePath': {'type': 'filename'}, @@ -36,7 +44,8 @@ iocMispMapping = { 'RouteEntryItem/Destination': {'type': 'ip-dst'}, 'Network/UserAgent': {'type': 'user-agent'}, - 'PortItem/localIP': {'type': 'ip-dst'}, + 'PortItem/localIP': {'type': 'ip-src'}, + 'PortItem/remoteIP': {'type': 'ip-dst'}, 'ProcessItem/name': {'type': 'pattern-in-memory', 'comment': 'ProcessName.'}, 'ProcessItem/path': {'type': 'pattern-in-memory', 'comment': 'ProcessPath.'}, @@ -115,13 +124,13 @@ def set_all_attributes(openioc, misp_event): continue else: continue - value = extract_field(openioc, 'Content') + value = extract_field(item, 'Content') if value: attribute_values['value'] = value else: # No value, ignoring continue - comment = extract_field(openioc, 'Comment') + comment = extract_field(item, 'Comment') if comment: attribute_values["comment"] = '{} {}'.format(attribute_values["comment"], comment) misp_event.add_attribute(**attribute_values) From 6885779be5424e78976aa40b10aa785df37f0461 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 15 Nov 2016 10:21:49 +0100 Subject: [PATCH 169/223] Allow to set org_id and orgc_id when creating a new MISPEvent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Squashed commit of the following: commit 8a5dfda8a728d8722dfba890729066948e744e44 Merge: 477f81d 332fc05 Author: Raphaël Vinot Date: Tue Nov 15 09:52:26 2016 +0100 Merge https://github.com/garanews/PyMISP into garanews-master commit 332fc0508d3f5dc32cf962bd2cad1d800b6b4b02 Author: garanews Date: Mon Nov 14 15:29:04 2016 +0100 Delete api_old.py commit 7fb955fa2fba964ae7120f4cf56cf85e203efe96 Author: garanews Date: Mon Nov 14 14:05:22 2016 +0100 Create api.py commit 7f6f45d9ce8ec4e948c5b2513b9bc59296985e09 Author: garanews Date: Mon Nov 14 14:05:06 2016 +0100 Rename api.py to api_old.py commit 63c325816c2bda646f462a322eb3063788ab91a4 Author: garanews Date: Mon Nov 14 11:48:17 2016 +0100 Add files via upload commit edf0cd1d90e89c30a4d3ef756d6bc6501de3885f Author: garanews Date: Wed Nov 2 15:31:42 2016 +0100 Add files via upload --- pymisp/api.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 1468c38..dd61efb 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -297,10 +297,10 @@ class PyMISP(object): # ######### Event handling (Json only) ######### # ############################################## - def _prepare_full_event(self, distribution, threat_level_id, analysis, info, date=None, published=False): + def _prepare_full_event(self, distribution, threat_level_id, analysis, info, date=None, published=False, orgc_id=None, org_id=None): misp_event = MISPEvent(self.describe_types) misp_event.set_all_values(info=info, distribution=distribution, threat_level_id=threat_level_id, - analysis=analysis, date=date) + analysis=analysis, date=date, orgc_id=orgc_id, org_id=org_id) if published: misp_event.publish() return misp_event @@ -341,8 +341,8 @@ class PyMISP(object): e.threat_level_id = threat_level_id return self.update_event(event['Event']['id'], json.dumps(e, cls=EncodeUpdate)) - def new_event(self, distribution=None, threat_level_id=None, analysis=None, info=None, date=None, published=False): - misp_event = self._prepare_full_event(distribution, threat_level_id, analysis, info, date, published) + def new_event(self, distribution=None, threat_level_id=None, analysis=None, info=None, date=None, published=False, orgc_id=None, org_id=None): + misp_event = self._prepare_full_event(distribution, threat_level_id, analysis, info, date, published, orgc_id, org_id) return self.add_event(json.dumps(misp_event, cls=EncodeUpdate)) def add_tag(self, event, tag): From 5632959f02444c177e03903deb6536bd2b274044 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 15 Nov 2016 10:59:29 +0100 Subject: [PATCH 170/223] Set user parameters in a function --- pymisp/api.py | 128 +++++++++++++++++++++----------------------------- 1 file changed, 53 insertions(+), 75 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index dd61efb..d05b4f8 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1098,6 +1098,47 @@ class PyMISP(object): # ############## Users ################## + def _set_user_parameters(self, email, org_id, role_id, password, external_auth_required, + external_auth_key, enable_password, nids_sid, server_id, + gpgkey, certif_public, autoalert, contactalert, disabled, + change_pw, termsaccepted, newsread): + user = {} + if email is not None: + user['email'] = email + if org_id is not None: + user['org_id'] = org_id + if role_id is not None: + user['role_id'] = role_id + if password is not None: + user['password'] = password + if external_auth_required is not None: + user['external_auth_required'] = external_auth_required + if external_auth_key is not None: + user['external_auth_key'] = external_auth_key + if enable_password is not None: + user['enable_password'] = enable_password + if nids_sid is not None: + user['nids_sid'] = nids_sid + if server_id is not None: + user['server_id'] = server_id + if gpgkey is not None: + user['gpgkey'] = gpgkey + if certif_public is not None: + user['certif_public'] = certif_public + if autoalert is not None: + user['autoalert'] = autoalert + if contactalert is not None: + user['contactalert'] = contactalert + if disabled is not None: + user['disabled'] = disabled + if change_pw is not None: + user['change_pw'] = change_pw + if termsaccepted is not None: + user['termsaccepted'] = termsaccepted + if newsread is not None: + user['newsread'] = newsread + return user + def get_users_list(self): session = self.__prepare_session() url = urljoin(self.root_url, 'admin/users') @@ -1110,52 +1151,18 @@ class PyMISP(object): response = session.get(url) return self._check_response(response) - def add_user_json(self, json_file): - session = self.__prepare_session() - jdata = json.load(open(json_file)) - url = urljoin(self.root_url, 'admin/users/add/') - response = session.post(url, data=json.dumps(jdata)) - return self._check_response(response) - def add_user(self, email, org_id, role_id, password=None, external_auth_required=None, external_auth_key=None, enable_password=None, nids_sid=None, server_id=None, gpgkey=None, certif_public=None, autoalert=None, contactalert=None, disabled=None, change_pw=None, termsaccepted=None, newsread=None): - new_user = {} - new_user['email'] = email - new_user['org_id'] = org_id - new_user['role_id'] = role_id - if password is not None: - new_user['password'] = password - if external_auth_required is not None: - new_user['external_auth_required'] = external_auth_required - if external_auth_key is not None: - new_user['external_auth_key'] = external_auth_key - if enable_password is not None: - new_user['enable_password'] = enable_password - if nids_sid is not None: - new_user['nids_sid'] = nids_sid - if server_id is not None: - new_user['server_id'] = server_id - if gpgkey is not None: - new_user['gpgkey'] = gpgkey - if certif_public is not None: - new_user['certif_public'] = certif_public - if autoalert is not None: - new_user['autoalert'] = autoalert - if contactalert is not None: - new_user['contactalert'] = contactalert - if disabled is not None: - new_user['disabled'] = disabled - if change_pw is not None: - new_user['change_pw'] = change_pw - if termsaccepted is not None: - new_user['termsaccepted'] = termsaccepted - if newsread is not None: - new_user['newsread'] = newsread - + new_user = self._set_user_parameters(email, org_id, role_id, password, + external_auth_required, external_auth_key, + enable_password, nids_sid, server_id, + gpgkey, certif_public, autoalert, + contactalert, disabled, change_pw, + termsaccepted, newsread) session = self.__prepare_session() url = urljoin(self.root_url, 'admin/users/add/') response = session.post(url, data=json.dumps(new_user)) @@ -1180,41 +1187,12 @@ class PyMISP(object): server_id=None, gpgkey=None, certif_public=None, autoalert=None, contactalert=None, disabled=None, change_pw=None, termsaccepted=None, newsread=None): - edit_user = {} - if email is not None: - edit_user['email'] = email - if org_id is not None: - edit_user['org_id'] = org_id - if role_id is not None: - edit_user['role_id'] = role_id - if password is not None: - edit_user['password'] = password - if external_auth_required is not None: - edit_user['external_auth_required'] = external_auth_required - if external_auth_key is not None: - edit_user['external_auth_key'] = external_auth_key - if enable_password is not None: - edit_user['enable_password'] = enable_password - if nids_sid is not None: - edit_user['nids_sid'] = nids_sid - if server_id is not None: - edit_user['server_id'] = server_id - if gpgkey is not None: - edit_user['gpgkey'] = gpgkey - if certif_public is not None: - edit_user['certif_public'] = certif_public - if autoalert is not None: - edit_user['autoalert'] = autoalert - if contactalert is not None: - edit_user['contactalert'] = contactalert - if disabled is not None: - edit_user['disabled'] = disabled - if change_pw is not None: - edit_user['change_pw'] = change_pw - if termsaccepted is not None: - edit_user['termsaccepted'] = termsaccepted - if newsread is not None: - edit_user['newsread'] = newsread + edit_user = self._set_user_parameters(email, org_id, role_id, password, + external_auth_required, external_auth_key, + enable_password, nids_sid, server_id, + gpgkey, certif_public, autoalert, + contactalert, disabled, change_pw, + termsaccepted, newsread) session = self.__prepare_session() url = urljoin(self.root_url, 'admin/users/edit/{}'.format(user_id)) From b6799f66a01aff7f56e4cc25e3e6b99c2abdfa63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 15 Nov 2016 11:03:51 +0100 Subject: [PATCH 171/223] Fix documentation generation --- docs/source/conf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 3984a79..eeecc5e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -41,7 +41,6 @@ extensions = [ 'sphinx.ext.mathjax', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode', - 'sphinx.ext.githubpages', 'sphinx.ext.napoleon', ] From 0c2a1f0d50ea025f138c427239d10ff728d3c737 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 16 Nov 2016 16:35:06 +0100 Subject: [PATCH 172/223] Update missing dependency exception --- pymisp/tools/stix.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymisp/tools/stix.py b/pymisp/tools/stix.py index b964295..fe9ca2a 100644 --- a/pymisp/tools/stix.py +++ b/pymisp/tools/stix.py @@ -12,7 +12,7 @@ except ImportError: def load_stix(stix, distribution=3, threat_level_id=2, analysis=0): '''Returns a MISPEvent object from a STIX package''' if not has_misp_stix_converter: - raise Exception('You need to install misp_stix_converter from https://github.com/MISP/MISP-STIX-Converter') + raise Exception('You need to install misp_stix_converter: pip install git+https://github.com/MISP/MISP-STIX-Converter.git') stix = open_stix(stix) return buildEvent(stix, distribution=distribution, threat_level_id=threat_level_id, analysis=analysis) @@ -25,7 +25,7 @@ def make_stix_package(misp_event, to_json=False, to_xml=False): ''' if not has_misp_stix_converter: - raise Exception('You need to install misp_stix_converter from https://github.com/MISP/MISP-STIX-Converter') + raise Exception('You need to install misp_stix_converter: pip install git+https://github.com/MISP/MISP-STIX-Converter.git') package = MISPtoSTIX(misp_event) if to_json: return package.to_json() From fba21ac051ff2fc8d5b3c93831b844a5c98f1a27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 16 Nov 2016 16:39:55 +0100 Subject: [PATCH 173/223] Version bump --- pymisp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index dac6257..4aad873 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.4.53' +__version__ = '2.4.54' from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey from .api import PyMISP From 35a4dd52bc18db7418eef58e41d97960ee589886 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 17 Nov 2016 17:07:29 +0100 Subject: [PATCH 174/223] Add signing support for MISP events --- examples/test_sign.py | 19 ++++++++ pymisp/mispevent.py | 101 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 117 insertions(+), 3 deletions(-) create mode 100755 examples/test_sign.py diff --git a/examples/test_sign.py b/examples/test_sign.py new file mode 100755 index 0000000..106aa29 --- /dev/null +++ b/examples/test_sign.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import argparse + +from pymisp import mispevent + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Sign & verify a MISP event.') + parser.add_argument("-i", "--input", required=True, help="Json file") + parser.add_argument("-u", "--uid", required=True, help="GPG UID") + args = parser.parse_args() + + me = mispevent.MISPEvent() + me.load(args.input) + + me.sign(args.uid) + me.verify(args.uid) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index e0ecb73..fe384d0 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -7,6 +7,7 @@ import json from json import JSONEncoder import os import warnings +import base64 try: from dateutil.parser import parse except ImportError: @@ -17,6 +18,19 @@ try: except ImportError: pass +try: + # pyme renamed to gpg the 2016-10-28 + import gpg + from gpg.constants.sig import mode + has_pyme = True +except ImportError: + # pyme renamed to gpg the 2016-10-28 + import pyme as gpg + from pyme.constants.sig import mode + has_pyme = True +except ImportError: + has_pyme = False + from .exceptions import PyMISPError, NewEventError, NewAttributeError # Least dirty way to support python 2 and 3 @@ -53,9 +67,33 @@ class MISPAttribute(object): self.timestamp = None self.sharing_group_id = None self.deleted = None + self.sig = None self.SharingGroup = [] self.ShadowAttribute = [] + def _serialize(self): + return '{type}{category}{to_ids}{uuid}{timestamp}{comment}{deleted}{value}'.format( + type=self.type, category=self.category, to_ids=self.to_ids, uuid=self.uuid, timestamp=self.timestamp, + comment=self.comment, deleted=self.deleted, value=self.value).encode() + + def sign(self, gpg_uid): + if not has_pyme: + raise Exception('pyme is required, please install: pip install --pre pyme3. You will also need libgpg-error-dev and libgpgme11-dev.') + to_sign = self._serialize() + with gpg.Context() as c: + keys = list(c.keylist(gpg_uid)) + c.signers = keys[:1] + signed, _ = c.sign(to_sign, mode=mode.DETACH) + self.sig = base64.b64encode(signed).decode() + + def verify(self, gpg_uid): + if not has_pyme: + raise Exception('pyme is required, please install: pip install --pre pyme3. You will also need libgpg-error-dev and libgpgme11-dev.') + signed_data = self._serialize() + with gpg.Context() as c: + keys = list(c.keylist(gpg_uid)) + c.verify(signed_data, signature=base64.b64decode(self.sig), verify=keys[:1]) + def set_all_values(self, **kwargs): if kwargs.get('type') and kwargs.get('category'): if kwargs['type'] not in self.category_type_mapping[kwargs['category']]: @@ -65,14 +103,14 @@ class MISPAttribute(object): self.type = kwargs['type'] if self.type not in self.types: raise NewAttributeError('{} is invalid, type has to be in {}'.format(self.type, (', '.join(self.types)))) - else: + elif not self.type: raise NewAttributeError('The type of the attribute is required.') type_defaults = self.sane_default[self.type] if kwargs.get('value'): self.value = kwargs['value'] - else: + elif not self.value: raise NewAttributeError('The value of the attribute is required.') # Default values @@ -111,6 +149,8 @@ class MISPAttribute(object): self.SharingGroup = kwargs['SharingGroup'] if kwargs.get('ShadowAttribute'): self.ShadowAttribute = kwargs['ShadowAttribute'] + if kwargs.get('sig'): + self.sig = kwargs['sig'] def _json(self): to_return = {'type': self.type, 'category': self.category, 'to_ids': self.to_ids, @@ -118,6 +158,8 @@ class MISPAttribute(object): 'comment': self.comment} if self.sharing_group_id: to_return['sharing_group_id'] = self.sharing_group_id + if self.sig: + to_return['sig'] = self.sig to_return = _int_to_str(to_return) return to_return @@ -193,6 +235,8 @@ class MISPEvent(object): self.attributes = [] # All other keys + self.sig = None + self.global_sig = None self.id = None self.orgc_id = None self.org_id = None @@ -209,6 +253,49 @@ class MISPEvent(object): self.RelatedEvent = [] self.Tag = [] + def _serialize(self): + return '{date}{threat_level_id}{info}{uuid}{analysis}{timestamp}'.format( + date=self.date, threat_level_id=self.threat_level_id, info=self.info, + uuid=self.uuid, analysis=self.analysis, timestamp=self.timestamp).encode() + + def _serialize_sigs(self): + all_sigs = self.sig + for a in self.attributes: + all_sigs += a.sig + return all_sigs.encode() + + def sign(self, gpg_uid): + if not has_pyme: + raise Exception('pyme is required, please install: pip install --pre pyme3. You will also need libgpg-error-dev and libgpgme11-dev.') + to_sign = self._serialize() + with gpg.Context() as c: + keys = list(c.keylist(gpg_uid)) + c.signers = keys[:1] + signed, _ = c.sign(to_sign, mode=mode.DETACH) + self.sig = base64.b64encode(signed).decode() + for a in self.attributes: + a.sign(gpg_uid) + to_sign_global = self._serialize_sigs() + with gpg.Context() as c: + keys = list(c.keylist(gpg_uid)) + c.signers = keys[:1] + signed, _ = c.sign(to_sign_global, mode=mode.DETACH) + self.global_sig = base64.b64encode(signed).decode() + + def verify(self, gpg_uid): + if not has_pyme: + raise Exception('pyme is required, please install: pip install --pre pyme3. You will also need libgpg-error-dev and libgpgme11-dev.') + signed_data = self._serialize() + with gpg.Context() as c: + keys = list(c.keylist(gpg_uid)) + c.verify(signed_data, signature=base64.b64decode(self.sig), verify=keys[:1]) + for a in self.attributes: + a.verify(gpg_uid) + to_verify_global = self._serialize_sigs() + with gpg.Context() as c: + keys = list(c.keylist(gpg_uid)) + c.verify(to_verify_global, signature=base64.b64decode(self.global_sig), verify=keys[:1]) + def load(self, json_event): self.new = False self.dump_full = True @@ -251,7 +338,7 @@ class MISPEvent(object): # Required value if kwargs.get('info'): self.info = kwargs['info'] - else: + elif not self.info: raise NewAttributeError('The info field of the new event is required.') # Default values for a valid event to send to a MISP instance @@ -308,6 +395,10 @@ class MISPEvent(object): self.RelatedEvent = kwargs['RelatedEvent'] if kwargs.get('Tag'): self.Tag = kwargs['Tag'] + if kwargs.get('sig'): + self.sig = kwargs['sig'] + if kwargs.get('global_sig'): + self.global_sig = kwargs['global_sig'] def _json(self): to_return = {'Event': {}} @@ -315,6 +406,10 @@ class MISPEvent(object): 'date': self.date.isoformat(), 'published': self.published, 'threat_level_id': self.threat_level_id, 'analysis': self.analysis, 'Attribute': []} + if self.sig: + to_return['sig'] = self.sig + if self.global_sig: + to_return['global_sig'] = self.global_sig if self.id: to_return['Event']['id'] = self.id if self.orgc_id: From 419897aba01a142551f948edc14dc6ce8229f84b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 17 Nov 2016 17:29:54 +0100 Subject: [PATCH 175/223] Bump recommended python3 version --- pymisp/api.py | 4 ++-- pymisp/mispevent.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index d05b4f8..93082b1 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -15,7 +15,7 @@ try: from urllib.parse import urljoin except ImportError: from urlparse import urljoin - warnings.warn("You're using python 2, it is strongly recommended to use python >=3.3") + warnings.warn("You're using python 2, it is strongly recommended to use python >=3.4") from io import BytesIO import zipfile @@ -33,7 +33,7 @@ from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate # Least dirty way to support python 2 and 3 try: basestring - warnings.warn("You're using python 2, it is strongly recommended to use python >=3.3") + warnings.warn("You're using python 2, it is strongly recommended to use python >=3.4") except NameError: basestring = str diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index fe384d0..c6ad4bb 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -37,7 +37,7 @@ from .exceptions import PyMISPError, NewEventError, NewAttributeError try: basestring unicode - warnings.warn("You're using python 2, it is strongly recommended to use python >=3.3") + warnings.warn("You're using python 2, it is strongly recommended to use python >=3.4") except NameError: basestring = str unicode = str From b8a391d26399fcdcbf5238b8e59ff409bdf204ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 17 Nov 2016 17:30:17 +0100 Subject: [PATCH 176/223] Avoid error if pyme3 isn't installed --- pymisp/mispevent.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index c6ad4bb..c4a5823 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -24,12 +24,13 @@ try: from gpg.constants.sig import mode has_pyme = True except ImportError: - # pyme renamed to gpg the 2016-10-28 - import pyme as gpg - from pyme.constants.sig import mode - has_pyme = True -except ImportError: - has_pyme = False + try: + # pyme renamed to gpg the 2016-10-28 + import pyme as gpg + from pyme.constants.sig import mode + has_pyme = True + except ImportError: + has_pyme = False from .exceptions import PyMISPError, NewEventError, NewAttributeError From d2e4e73abb1f63283943cb233711d7bc3b450655 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 18 Nov 2016 18:01:57 +0100 Subject: [PATCH 177/223] Allow to pass a passphrase as parameter for signing. --- pymisp/mispevent.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index c4a5823..829d8da 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -77,13 +77,15 @@ class MISPAttribute(object): type=self.type, category=self.category, to_ids=self.to_ids, uuid=self.uuid, timestamp=self.timestamp, comment=self.comment, deleted=self.deleted, value=self.value).encode() - def sign(self, gpg_uid): + def sign(self, gpg_uid, passphrase=None): if not has_pyme: raise Exception('pyme is required, please install: pip install --pre pyme3. You will also need libgpg-error-dev and libgpgme11-dev.') to_sign = self._serialize() with gpg.Context() as c: keys = list(c.keylist(gpg_uid)) c.signers = keys[:1] + if passphrase: + c.set_passphrase_cb(lambda *args: passphrase) signed, _ = c.sign(to_sign, mode=mode.DETACH) self.sig = base64.b64encode(signed).decode() @@ -265,21 +267,25 @@ class MISPEvent(object): all_sigs += a.sig return all_sigs.encode() - def sign(self, gpg_uid): + def sign(self, gpg_uid, passphrase=None): if not has_pyme: raise Exception('pyme is required, please install: pip install --pre pyme3. You will also need libgpg-error-dev and libgpgme11-dev.') to_sign = self._serialize() with gpg.Context() as c: keys = list(c.keylist(gpg_uid)) c.signers = keys[:1] + if passphrase: + c.set_passphrase_cb(lambda *args: passphrase) signed, _ = c.sign(to_sign, mode=mode.DETACH) self.sig = base64.b64encode(signed).decode() for a in self.attributes: - a.sign(gpg_uid) + a.sign(gpg_uid, passphrase) to_sign_global = self._serialize_sigs() with gpg.Context() as c: keys = list(c.keylist(gpg_uid)) c.signers = keys[:1] + if passphrase: + c.set_passphrase_cb(lambda *args: passphrase) signed, _ = c.sign(to_sign_global, mode=mode.DETACH) self.global_sig = base64.b64encode(signed).decode() From f62cf9f3d93fbfc3c210606f166e33d3adfbdaf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 21 Nov 2016 10:44:03 +0100 Subject: [PATCH 178/223] More granularity in the verification --- pymisp/mispevent.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 829d8da..1fdd0cf 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -95,7 +95,11 @@ class MISPAttribute(object): signed_data = self._serialize() with gpg.Context() as c: keys = list(c.keylist(gpg_uid)) - c.verify(signed_data, signature=base64.b64decode(self.sig), verify=keys[:1]) + try: + c.verify(signed_data, signature=base64.b64decode(self.sig), verify=keys[:1]) + return {self.uuid: True} + except: + return {self.uuid: False} def set_all_values(self, **kwargs): if kwargs.get('type') and kwargs.get('category'): @@ -292,16 +296,26 @@ class MISPEvent(object): def verify(self, gpg_uid): if not has_pyme: raise Exception('pyme is required, please install: pip install --pre pyme3. You will also need libgpg-error-dev and libgpgme11-dev.') + to_return = {} signed_data = self._serialize() with gpg.Context() as c: keys = list(c.keylist(gpg_uid)) - c.verify(signed_data, signature=base64.b64decode(self.sig), verify=keys[:1]) + try: + c.verify(signed_data, signature=base64.b64decode(self.sig), verify=keys[:1]) + to_return[self.uuid] = True + except: + to_return[self.uuid] = False for a in self.attributes: - a.verify(gpg_uid) + to_return.update(a.verify(gpg_uid)) to_verify_global = self._serialize_sigs() with gpg.Context() as c: keys = list(c.keylist(gpg_uid)) - c.verify(to_verify_global, signature=base64.b64decode(self.global_sig), verify=keys[:1]) + try: + c.verify(to_verify_global, signature=base64.b64decode(self.global_sig), verify=keys[:1]) + to_return['global'] = True + except: + to_return['global'] = False + return to_return def load(self, json_event): self.new = False From f7b228ac563c4ac54adc7b2b8f6721d4f41653e0 Mon Sep 17 00:00:00 2001 From: Hannah Ward Date: Mon, 21 Nov 2016 11:57:49 +0000 Subject: [PATCH 179/223] Use misp_stix_converter.converters.convert's load_stix method --- pymisp/tools/stix.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pymisp/tools/stix.py b/pymisp/tools/stix.py index fe9ca2a..b6463c8 100644 --- a/pymisp/tools/stix.py +++ b/pymisp/tools/stix.py @@ -2,7 +2,8 @@ # -*- coding: utf-8 -*- try: - from misp_stix_converter.converters.buildMISPAttribute import buildEvent, open_stix + from misp_stix_converter.converters.buildMISPAttribute import buildEvent + from misp_stix_converter.converters import convert from misp_stix_converter.converters.convert import MISPtoSTIX has_misp_stix_converter = True except ImportError: @@ -13,7 +14,7 @@ def load_stix(stix, distribution=3, threat_level_id=2, analysis=0): '''Returns a MISPEvent object from a STIX package''' if not has_misp_stix_converter: raise Exception('You need to install misp_stix_converter: pip install git+https://github.com/MISP/MISP-STIX-Converter.git') - stix = open_stix(stix) + stix = convert.load_stix(stix) return buildEvent(stix, distribution=distribution, threat_level_id=threat_level_id, analysis=analysis) From c90c73c9afce2f87e1f6dc2a615f3ef91871ef3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 24 Nov 2016 10:41:47 +0100 Subject: [PATCH 180/223] Improve debug mode --- pymisp/api.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pymisp/api.py b/pymisp/api.py index 93082b1..6f4acf8 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -755,6 +755,9 @@ class PyMISP(object): raise Exception('Invalid controller. Can only be {}'.format(', '.join(['events', 'attributes']))) url = urljoin(self.root_url, '{}/{}'.format(controller, path.lstrip('/'))) query = {'request': query} + if self.debug: + print('URL: ', url) + print('Query: ', query) response = session.post(url, data=json.dumps(query)) return self._check_response(response) From 7b5b45146c6fc3ab005af92d1113938d1d77a15f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 24 Nov 2016 10:50:46 +0100 Subject: [PATCH 181/223] Remove crazy replace --- pymisp/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 6f4acf8..a457ca2 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -854,8 +854,8 @@ class PyMISP(object): :param metadata: return onlymetadata if True """ - val = self.__prepare_rest_search(values, not_values).replace('/', '|') - tag = self.__prepare_rest_search(tags, not_tags).replace(':', ';') + val = self.__prepare_rest_search(values, not_values) + tag = self.__prepare_rest_search(tags, not_tags) query = {} if len(val) != 0: query['value'] = val From d3d7bccf0b78e3e7226a0dfd1681b31500552e07 Mon Sep 17 00:00:00 2001 From: Nicolas Bareil Date: Tue, 29 Nov 2016 09:14:18 +0100 Subject: [PATCH 182/223] Factorize all add_* in favor of add_named_attribute() Not tested... --- pymisp/api.py | 213 +++++++++++++++----------------------------------- 1 file changed, 62 insertions(+), 151 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index a457ca2..92ecd1d 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -399,22 +399,13 @@ class PyMISP(object): return self._send_attributes(event, attributes, proposal) def av_detection_link(self, event, link, category='Antivirus detection', to_ids=False, comment=None, distribution=None, proposal=False): - attributes = [] - for link in self._one_or_more(link): - attributes.append(self._prepare_full_attribute(category, 'link', link, to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) + return self._add_named_attributes(event, 'link', link, category, to_ids, comment, distribution, proposal) def add_detection_name(self, event, name, category='Antivirus detection', to_ids=False, comment=None, distribution=None, proposal=False): - attributes = [] - for name in self._one_or_more(name): - attributes.append(self._prepare_full_attribute(category, 'text', name, to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) + return self._add_named_attributes(event, 'text', name, category, to_ids, comment, distribution, proposal) def add_filename(self, event, filename, category='Artifacts dropped', to_ids=False, comment=None, distribution=None, proposal=False): - attributes = [] - for filename in self._one_or_more(filename): - attributes.append(self._prepare_full_attribute(category, 'filename', filename, to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) + return self._add_named_attributes(event, 'filename', filename, category, to_ids, comment, distribution, proposal) def add_regkey(self, event, regkey, rvalue=None, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False): if rvalue: @@ -443,199 +434,119 @@ class PyMISP(object): return self._send_attributes(event, attributes, proposal) def add_pattern(self, event, pattern, in_file=True, in_memory=False, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False): - attributes = [] - for pattern in self._one_or_more(pattern): - if in_file: - attributes.append(self._prepare_full_attribute(category, 'pattern-in-file', pattern, to_ids, comment, distribution)) - if in_memory: - attributes.append(self._prepare_full_attribute(category, 'pattern-in-memory', pattern, to_ids, comment, distribution)) - - return self._send_attributes(event, attributes, proposal) + if not (in_file or in_memory): + raise PyMISPError('Invalid pattern type: please use in_memory=True or in_file=True') + itemtype = 'pattern-in-file' if in_file else 'pattern-in-memory' + return self._add_named_attributes(event, itemtype, pattern, category, to_ids, comment, distribution, proposal) def add_pipe(self, event, named_pipe, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False): - attributes = [] - for named_pipe in self._one_or_more(named_pipe): - if not named_pipe.startswith('\\.\\pipe\\'): - named_pipe = '\\.\\pipe\\{}'.format(named_pipe) - attributes.append(self._prepare_full_attribute(category, 'named pipe', named_pipe, to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) + def scrub(s): + if not s.startswith('\\.\\pipe\\'): + s = '\\.\\pipe\\{}'.format(s) + return s + attributes = map(scrub, self._one_or_more(named_pipe)) + return self._add_named_attributes(event, 'named pipe', value, category, to_ids, comment, distribution, proposal) def add_mutex(self, event, mutex, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False): - attributes = [] - if not mutex.startswith('\\BaseNamedObjects\\'): - mutex = '\\BaseNamedObjects\\{}'.format(mutex) - attributes.append(self._prepare_full_attribute(category, 'mutex', mutex, to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) + def scrub(s): + if not s.startswith('\\BaseNamedObjects\\'): + s = '\\BaseNamedObjects\\{}'.format(s) + return self + attributes = map(scrub, self._one_or_more(mutex)) + return self._add_named_attributes(event, 'mutex', attributes, category, to_ids, comment, distribution, proposal) def add_yara(self, event, yara, category='Payload delivery', to_ids=False, comment=None, distribution=None, proposal=False): - attributes = [] - for yara in self._one_or_more(yara): - attributes.append(self._prepare_full_attribute(category, 'yara', yara, to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) + return self._add_named_attributes(event, 'yara', yara, category, to_ids, comment, distribution, proposal) # ##### Network attributes ##### def add_ipdst(self, event, ipdst, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): - attributes = [] - for ipdst in self._one_or_more(ipdst): - attributes.append(self._prepare_full_attribute(category, 'ip-dst', ipdst, to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) + return self._add_named_attributes(event, 'ip-dst', ipdst, category, to_ids, comment, distribution, proposal) def add_ipsrc(self, event, ipsrc, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): - attributes = [] - for ipsrc in self._one_or_more(ipsrc): - attributes.append(self._prepare_full_attribute(category, 'ip-src', ipsrc, to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) + return self._add_named_attributes(event, 'ip-src', ipsrc, category, to_ids, comment, distribution, proposal) def add_hostname(self, event, hostname, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): - attributes = [] - for hostname in self._one_or_more(hostname): - attributes.append(self._prepare_full_attribute(category, 'hostname', hostname, to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) + return self._add_named_attributes(event, 'hostname', hostname, category, to_ids, comment, distribution, proposal) def add_domain(self, event, domain, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): - attributes = [] - for domain in self._one_or_more(domain): - attributes.append(self._prepare_full_attribute(category, 'domain', domain, to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) + return self._add_named_attributes(event, 'domain', domain, category, to_ids, comment, distribution, proposal) def add_domain_ip(self, event, domain, ip, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): - attributes = [] - attributes.append(self._prepare_full_attribute(category, 'domain|ip', "%s|%s" % (domain, ip), to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) + composed = map(lambda x,y: '%s|%s' % (x, z), zip(domain, ip)) + return self._add_named_attributes(event, 'domain|ip', composed, category, to_ids, comment, distribution, proposal) def add_domains_ips(self, event, domain_ips, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): - attributes = [] - for domain, ip in domain_ips.items(): - attributes.append(self._prepare_full_attribute(category, 'domain|ip', "%s|%s" % (domain, ip), to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) + composed = map(lambda x,y: '%s|%s' % (x, z), domain_ips.items()) + return self._add_named_attributes(event, 'domain|ip', composed, category, to_ids, comment, distribution, proposal) def add_url(self, event, url, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): - attributes = [] - for url in self._one_or_more(url): - attributes.append(self._prepare_full_attribute(category, 'url', url, to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) + return self._add_named_attributes(event, 'url', url, category, to_ids, comment, distribution, proposal) def add_useragent(self, event, useragent, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): - attributes = [] - for useragent in self._one_or_more(useragent): - attributes.append(self._prepare_full_attribute(category, 'user-agent', useragent, to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) + return self._add_named_attributes(event, 'user-agent', useragent, category, to_ids, comment, distribution, proposal) def add_traffic_pattern(self, event, pattern, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): - attributes = [] - for pattern in self._one_or_more(pattern): - attributes.append(self._prepare_full_attribute(category, 'pattern-in-traffic', pattern, to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) + return self._add_named_attributes(event, 'pattern-in-traffic', pattern, category, to_ids, comment, distribution, proposal) def add_snort(self, event, snort, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): - attributes = [] - for snort in self._one_or_more(snort): - attributes.append(self._prepare_full_attribute(category, 'snort', snort, to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) + return self._add_named_attributes(event, 'snort', snort, category, to_ids, comment, distribution, proposal) def add_net_other(self, event, netother, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): - attributes = [] - attributes.append(self._prepare_full_attribute(category, 'other', netother, to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) + return self._add_named_attributes(event, 'other', netother, category, to_ids, comment, distribution, proposal) # ##### Email attributes ##### - def add_email_src(self, event, email, to_ids=True, comment=None, distribution=None, proposal=False): - attributes = [] - for email in self._one_or_more(email): - attributes.append(self._prepare_full_attribute('Payload delivery', 'email-src', email, to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) + def add_email_src(self, event, email, category='Payload delivery', to_ids=True, comment=None, distribution=None, proposal=False): + return self._add_named_attributes(event, 'email-src', email, category, to_ids, comment, distribution, proposal) def add_email_dst(self, event, email, category='Payload delivery', to_ids=True, comment=None, distribution=None, proposal=False): - attributes = [] - for email in self._one_or_more(email): - attributes.append(self._prepare_full_attribute(category, 'email-dst', email, to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) + return self._add_named_attributes(event, 'email-dst', email, category, to_ids, comment, distribution, proposal) - def add_email_subject(self, event, email, to_ids=True, comment=None, distribution=None, proposal=False): - attributes = [] - for email in self._one_or_more(email): - attributes.append(self._prepare_full_attribute('Payload delivery', 'email-subject', email, to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) + def add_email_subject(self, event, email, category='Payload delivery', to_ids=True, comment=None, distribution=None, proposal=False): + return self._add_named_attributes(event, 'email-subject', email, category, to_ids, comment, distribution, proposal) - def add_email_attachment(self, event, email, to_ids=True, comment=None, distribution=None, proposal=False): - attributes = [] - for email in self._one_or_more(email): - attributes.append(self._prepare_full_attribute('Payload delivery', 'email-attachment', email, to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) + def add_email_attachment(self, event, email, category='Payload delivery', to_ids=True, comment=None, distribution=None, proposal=False): + return self._add_named_attributes(event, 'email-attachement', email, category, to_ids, comment, distribution, proposal) # ##### Target attributes ##### - def add_target_email(self, event, target, to_ids=True, comment=None, distribution=None, proposal=False): - attributes = [] - for target in self._one_or_more(target): - attributes.append(self._prepare_full_attribute('Targeting data', 'target-email', target, to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) + def add_target_email(self, event, target, category='Targeting data', to_ids=True, comment=None, distribution=None, proposal=False): + return self._add_named_attributes(event, 'target-email', target, category, to_ids, comment, distribution, proposal) - def add_target_user(self, event, target, to_ids=True, comment=None, distribution=None, proposal=False): - attributes = [] - for target in self._one_or_more(target): - attributes.append(self._prepare_full_attribute('Targeting data', 'target-user', target, to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) + def add_target_user(self, event, target, category='Targeting data', to_ids=True, comment=None, distribution=None, proposal=False): + return self._add_named_attributes(event, 'target-user', target, category, to_ids, comment, distribution, proposal) - def add_target_machine(self, event, target, to_ids=True, comment=None, distribution=None, proposal=False): - attributes = [] - for target in self._one_or_more(target): - attributes.append(self._prepare_full_attribute('Targeting data', 'target-machine', target, to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) + def add_target_machine(self, event, target, category='Targeting data', to_ids=True, comment=None, distribution=None, proposal=False): + return self._add_named_attributes(event, 'target-machine', target, category, to_ids, comment, distribution, proposal) - def add_target_org(self, event, target, to_ids=True, comment=None, distribution=None, proposal=False): - attributes = [] - for target in self._one_or_more(target): - attributes.append(self._prepare_full_attribute('Targeting data', 'target-org', target, to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) + def add_target_org(self, event, target, category='Targeting data', to_ids=True, comment=None, distribution=None, proposal=False): + return self._add_named_attributes(event, 'target-orge', target, category, to_ids, comment, distribution, proposal) - def add_target_location(self, event, target, to_ids=True, comment=None, distribution=None, proposal=False): - attributes = [] - for target in self._one_or_more(target): - attributes.append(self._prepare_full_attribute('Targeting data', 'target-location', target, to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) + def add_target_location(self, event, target, category='Targeting data', to_ids=True, comment=None, distribution=None, proposal=False): + return self._add_named_attributes(event, 'target-location', target, category, to_ids, comment, distribution, proposal) - def add_target_external(self, event, target, to_ids=True, comment=None, distribution=None, proposal=False): - attributes = [] - for target in self._one_or_more(target): - attributes.append(self._prepare_full_attribute('Targeting data', 'target-external', target, to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) + def add_target_external(self, event, target, category='Targeting data', to_ids=True, comment=None, distribution=None, proposal=False): + return self._add_named_attributes(event, 'target-external', target, category, to_ids, comment, distribution, proposal) # ##### Attribution attributes ##### - def add_threat_actor(self, event, target, to_ids=True, comment=None, distribution=None, proposal=False): - attributes = [] - for target in self._one_or_more(target): - attributes.append(self._prepare_full_attribute('Attribution', 'threat-actor', target, to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) + def add_threat_actor(self, event, target, category='Attribution', to_ids=True, comment=None, distribution=None, proposal=False): + return self._add_named_attributes(event, 'threat-actor', target, category, to_ids, comment, distribution, proposal) # ##### Internal reference attributes ##### - def add_internal_link(self, event, reference, to_ids=False, comment=None, distribution=None, proposal=False): - attributes = [] - for reference in self._one_or_more(reference): - attributes.append(self._prepare_full_attribute('Internal reference', 'link', reference, to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) + def add_internal_link(self, event, reference, category='Internal reference', to_ids=False, comment=None, distribution=None, proposal=False): + return self._add_named_attributes(event, 'link', reference, category, to_ids, comment, distribution, proposal) - def add_internal_comment(self, event, reference, to_ids=False, comment=None, distribution=None, proposal=False): - attributes = [] - for reference in self._one_or_more(reference): - attributes.append(self._prepare_full_attribute('Internal reference', 'comment', reference, to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) + def add_internal_comment(self, event, reference, category='Internal reference', to_ids=False, comment=None, distribution=None, proposal=False): + return self._add_named_attributes(event, 'comment', reference, category, to_ids, comment, distribution, proposal) - def add_internal_text(self, event, reference, to_ids=False, comment=None, distribution=None, proposal=False): - attributes = [] - for reference in self._one_or_more(reference): - attributes.append(self._prepare_full_attribute('Internal reference', 'text', reference, to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) + def add_internal_text(self, event, reference, category='Internal reference', to_ids=False, comment=None, distribution=None, proposal=False): + return self._add_named_attributes(event, 'text', reference, category, to_ids, comment, distribution, proposal) - def add_internal_other(self, event, reference, to_ids=False, comment=None, distribution=None, proposal=False): - attributes = [] - for reference in self._one_or_more(reference): - attributes.append(self._prepare_full_attribute('Internal reference', 'other', reference, to_ids, comment, distribution)) - return self._send_attributes(event, attributes, proposal) + def add_internal_other(self, event, reference, category='Internal reference', to_ids=False, comment=None, distribution=None, proposal=False): + return self._add_named_attributes(event, 'other', reference, category, to_ids, comment, distribution, proposal) # ################################################## # ######### Upload samples through the API ######### From f956fd526e99825d476c15e59d59af5b85168016 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 30 Nov 2016 17:14:55 +0100 Subject: [PATCH 183/223] Fix neo4j --- pymisp/tools/neo4j.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymisp/tools/neo4j.py b/pymisp/tools/neo4j.py index af5ba2c..42f5214 100644 --- a/pymisp/tools/neo4j.py +++ b/pymisp/tools/neo4j.py @@ -18,7 +18,7 @@ class Neo4j(): if not has_py2neo: raise Exception('py2neo is required, please install: pip install py2neo') authenticate(host, username, password) - self.graph = Graph() + self.graph = Graph("http://{}/db/data/".format(host)) def load_events_directory(self, directory): self.events = [] @@ -54,5 +54,5 @@ class Neo4j(): av = Relationship(attr_node, "is", val) s = val | ev | av tx.merge(s) - tx.graph.push(s) + #tx.graph.push(s) tx.commit() From e44e33fe903d0d3af551e4895f72d784269b2def Mon Sep 17 00:00:00 2001 From: Nicolas Bareil Date: Thu, 1 Dec 2016 10:48:33 +0100 Subject: [PATCH 184/223] capitalizeformat() does not exist on Python2 and fix category variables --- pymisp/mispevent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 1fdd0cf..2223868 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -104,7 +104,7 @@ class MISPAttribute(object): def set_all_values(self, **kwargs): if kwargs.get('type') and kwargs.get('category'): if kwargs['type'] not in self.category_type_mapping[kwargs['category']]: - raise NewAttributeError('{} and {} is an invalid combinaison, type for this category has to be in {}'.capitalizeformat(self.type, self.category, (', '.join(self.category_type_mapping[self.category])))) + raise NewAttributeError('{} and {} is an invalid combinaison, type for this category has to be in {}'.format(self.type, self.category, (', '.join(self.category_type_mapping[kwargs['category']])))) # Required if kwargs.get('type'): self.type = kwargs['type'] From fff3a66d09d2c8eb19d3ed7cdc920b6de9a33892 Mon Sep 17 00:00:00 2001 From: Nicolas Bareil Date: Thu, 1 Dec 2016 10:49:12 +0100 Subject: [PATCH 185/223] Unit-tests --- pymisp/api.py | 72 +++++++++++++++++++++---------------------- tests/test_offline.py | 72 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 36 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 92ecd1d..55d8852 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -399,13 +399,13 @@ class PyMISP(object): return self._send_attributes(event, attributes, proposal) def av_detection_link(self, event, link, category='Antivirus detection', to_ids=False, comment=None, distribution=None, proposal=False): - return self._add_named_attributes(event, 'link', link, category, to_ids, comment, distribution, proposal) + return self.add_named_attribute(event, 'link', link, category, to_ids, comment, distribution, proposal) def add_detection_name(self, event, name, category='Antivirus detection', to_ids=False, comment=None, distribution=None, proposal=False): - return self._add_named_attributes(event, 'text', name, category, to_ids, comment, distribution, proposal) + return self.add_named_attribute(event, 'text', name, category, to_ids, comment, distribution, proposal) def add_filename(self, event, filename, category='Artifacts dropped', to_ids=False, comment=None, distribution=None, proposal=False): - return self._add_named_attributes(event, 'filename', filename, category, to_ids, comment, distribution, proposal) + return self.add_named_attribute(event, 'filename', filename, category, to_ids, comment, distribution, proposal) def add_regkey(self, event, regkey, rvalue=None, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False): if rvalue: @@ -423,7 +423,7 @@ class PyMISP(object): attributes = [] for regkey, rvalue in regkeys_values.items(): - if rvalue: + if rvalue is not None: type_value = 'regkey|value' value = '{}|{}'.format(regkey, rvalue) else: @@ -437,7 +437,7 @@ class PyMISP(object): if not (in_file or in_memory): raise PyMISPError('Invalid pattern type: please use in_memory=True or in_file=True') itemtype = 'pattern-in-file' if in_file else 'pattern-in-memory' - return self._add_named_attributes(event, itemtype, pattern, category, to_ids, comment, distribution, proposal) + return self.add_named_attribute(event, itemtype, pattern, category, to_ids, comment, distribution, proposal) def add_pipe(self, event, named_pipe, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False): def scrub(s): @@ -445,7 +445,7 @@ class PyMISP(object): s = '\\.\\pipe\\{}'.format(s) return s attributes = map(scrub, self._one_or_more(named_pipe)) - return self._add_named_attributes(event, 'named pipe', value, category, to_ids, comment, distribution, proposal) + return self.add_named_attribute(event, 'named pipe', attributes, category, to_ids, comment, distribution, proposal) def add_mutex(self, event, mutex, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False): def scrub(s): @@ -453,100 +453,100 @@ class PyMISP(object): s = '\\BaseNamedObjects\\{}'.format(s) return self attributes = map(scrub, self._one_or_more(mutex)) - return self._add_named_attributes(event, 'mutex', attributes, category, to_ids, comment, distribution, proposal) + return self.add_named_attribute(event, 'mutex', attributes, category, to_ids, comment, distribution, proposal) def add_yara(self, event, yara, category='Payload delivery', to_ids=False, comment=None, distribution=None, proposal=False): - return self._add_named_attributes(event, 'yara', yara, category, to_ids, comment, distribution, proposal) + return self.add_named_attribute(event, 'yara', yara, category, to_ids, comment, distribution, proposal) # ##### Network attributes ##### def add_ipdst(self, event, ipdst, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): - return self._add_named_attributes(event, 'ip-dst', ipdst, category, to_ids, comment, distribution, proposal) + return self.add_named_attribute(event, 'ip-dst', ipdst, category, to_ids, comment, distribution, proposal) def add_ipsrc(self, event, ipsrc, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): - return self._add_named_attributes(event, 'ip-src', ipsrc, category, to_ids, comment, distribution, proposal) + return self.add_named_attribute(event, 'ip-src', ipsrc, category, to_ids, comment, distribution, proposal) def add_hostname(self, event, hostname, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): - return self._add_named_attributes(event, 'hostname', hostname, category, to_ids, comment, distribution, proposal) + return self.add_named_attribute(event, 'hostname', hostname, category, to_ids, comment, distribution, proposal) def add_domain(self, event, domain, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): - return self._add_named_attributes(event, 'domain', domain, category, to_ids, comment, distribution, proposal) + return self.add_named_attribute(event, 'domain', domain, category, to_ids, comment, distribution, proposal) def add_domain_ip(self, event, domain, ip, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): - composed = map(lambda x,y: '%s|%s' % (x, z), zip(domain, ip)) - return self._add_named_attributes(event, 'domain|ip', composed, category, to_ids, comment, distribution, proposal) + composed = map(lambda x: '%s|%s' % (domain, x), ip) + return self.add_named_attribute(event, 'domain|ip', composed, category, to_ids, comment, distribution, proposal) def add_domains_ips(self, event, domain_ips, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): - composed = map(lambda x,y: '%s|%s' % (x, z), domain_ips.items()) - return self._add_named_attributes(event, 'domain|ip', composed, category, to_ids, comment, distribution, proposal) + composed = map(lambda (x,y): '%s|%s' % (x, y), domain_ips.items()) + return self.add_named_attribute(event, 'domain|ip', composed, category, to_ids, comment, distribution, proposal) def add_url(self, event, url, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): - return self._add_named_attributes(event, 'url', url, category, to_ids, comment, distribution, proposal) + return self.add_named_attribute(event, 'url', url, category, to_ids, comment, distribution, proposal) def add_useragent(self, event, useragent, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): - return self._add_named_attributes(event, 'user-agent', useragent, category, to_ids, comment, distribution, proposal) + return self.add_named_attribute(event, 'user-agent', useragent, category, to_ids, comment, distribution, proposal) def add_traffic_pattern(self, event, pattern, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): - return self._add_named_attributes(event, 'pattern-in-traffic', pattern, category, to_ids, comment, distribution, proposal) + return self.add_named_attribute(event, 'pattern-in-traffic', pattern, category, to_ids, comment, distribution, proposal) def add_snort(self, event, snort, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): - return self._add_named_attributes(event, 'snort', snort, category, to_ids, comment, distribution, proposal) + return self.add_named_attribute(event, 'snort', snort, category, to_ids, comment, distribution, proposal) def add_net_other(self, event, netother, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): - return self._add_named_attributes(event, 'other', netother, category, to_ids, comment, distribution, proposal) + return self.add_named_attribute(event, 'other', netother, category, to_ids, comment, distribution, proposal) # ##### Email attributes ##### def add_email_src(self, event, email, category='Payload delivery', to_ids=True, comment=None, distribution=None, proposal=False): - return self._add_named_attributes(event, 'email-src', email, category, to_ids, comment, distribution, proposal) + return self.add_named_attribute(event, 'email-src', email, category, to_ids, comment, distribution, proposal) def add_email_dst(self, event, email, category='Payload delivery', to_ids=True, comment=None, distribution=None, proposal=False): - return self._add_named_attributes(event, 'email-dst', email, category, to_ids, comment, distribution, proposal) + return self.add_named_attribute(event, 'email-dst', email, category, to_ids, comment, distribution, proposal) def add_email_subject(self, event, email, category='Payload delivery', to_ids=True, comment=None, distribution=None, proposal=False): - return self._add_named_attributes(event, 'email-subject', email, category, to_ids, comment, distribution, proposal) + return self.add_named_attribute(event, 'email-subject', email, category, to_ids, comment, distribution, proposal) def add_email_attachment(self, event, email, category='Payload delivery', to_ids=True, comment=None, distribution=None, proposal=False): - return self._add_named_attributes(event, 'email-attachement', email, category, to_ids, comment, distribution, proposal) + return self.add_named_attribute(event, 'email-attachment', email, category, to_ids, comment, distribution, proposal) # ##### Target attributes ##### def add_target_email(self, event, target, category='Targeting data', to_ids=True, comment=None, distribution=None, proposal=False): - return self._add_named_attributes(event, 'target-email', target, category, to_ids, comment, distribution, proposal) + return self.add_named_attribute(event, 'target-email', target, category, to_ids, comment, distribution, proposal) def add_target_user(self, event, target, category='Targeting data', to_ids=True, comment=None, distribution=None, proposal=False): - return self._add_named_attributes(event, 'target-user', target, category, to_ids, comment, distribution, proposal) + return self.add_named_attribute(event, 'target-user', target, category, to_ids, comment, distribution, proposal) def add_target_machine(self, event, target, category='Targeting data', to_ids=True, comment=None, distribution=None, proposal=False): - return self._add_named_attributes(event, 'target-machine', target, category, to_ids, comment, distribution, proposal) + return self.add_named_attribute(event, 'target-machine', target, category, to_ids, comment, distribution, proposal) def add_target_org(self, event, target, category='Targeting data', to_ids=True, comment=None, distribution=None, proposal=False): - return self._add_named_attributes(event, 'target-orge', target, category, to_ids, comment, distribution, proposal) + return self.add_named_attribute(event, 'target-org', target, category, to_ids, comment, distribution, proposal) def add_target_location(self, event, target, category='Targeting data', to_ids=True, comment=None, distribution=None, proposal=False): - return self._add_named_attributes(event, 'target-location', target, category, to_ids, comment, distribution, proposal) + return self.add_named_attribute(event, 'target-location', target, category, to_ids, comment, distribution, proposal) def add_target_external(self, event, target, category='Targeting data', to_ids=True, comment=None, distribution=None, proposal=False): - return self._add_named_attributes(event, 'target-external', target, category, to_ids, comment, distribution, proposal) + return self.add_named_attribute(event, 'target-external', target, category, to_ids, comment, distribution, proposal) # ##### Attribution attributes ##### def add_threat_actor(self, event, target, category='Attribution', to_ids=True, comment=None, distribution=None, proposal=False): - return self._add_named_attributes(event, 'threat-actor', target, category, to_ids, comment, distribution, proposal) + return self.add_named_attribute(event, 'threat-actor', target, category, to_ids, comment, distribution, proposal) # ##### Internal reference attributes ##### def add_internal_link(self, event, reference, category='Internal reference', to_ids=False, comment=None, distribution=None, proposal=False): - return self._add_named_attributes(event, 'link', reference, category, to_ids, comment, distribution, proposal) + return self.add_named_attribute(event, 'link', reference, category, to_ids, comment, distribution, proposal) def add_internal_comment(self, event, reference, category='Internal reference', to_ids=False, comment=None, distribution=None, proposal=False): - return self._add_named_attributes(event, 'comment', reference, category, to_ids, comment, distribution, proposal) + return self.add_named_attribute(event, 'comment', reference, category, to_ids, comment, distribution, proposal) def add_internal_text(self, event, reference, category='Internal reference', to_ids=False, comment=None, distribution=None, proposal=False): - return self._add_named_attributes(event, 'text', reference, category, to_ids, comment, distribution, proposal) + return self.add_named_attribute(event, 'text', reference, category, to_ids, comment, distribution, proposal) def add_internal_other(self, event, reference, category='Internal reference', to_ids=False, comment=None, distribution=None, proposal=False): - return self._add_named_attributes(event, 'other', reference, category, to_ids, comment, distribution, proposal) + return self.add_named_attribute(event, 'other', reference, category, to_ids, comment, distribution, proposal) # ################################################## # ######### Upload samples through the API ######### diff --git a/tests/test_offline.py b/tests/test_offline.py index b4854af..c4a8a1a 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -126,5 +126,77 @@ class TestOffline(unittest.TestCase): json.dumps(misp_event, cls=EncodeUpdate) json.dumps(misp_event, cls=EncodeFull) + def test_addAttributes(self, m): + class MockPyMISP(PyMISP): + def _send_attributes(self, event, attributes, proposal=False): + return len(attributes) + self.initURI(m) + p = MockPyMISP(self.domain, self.key) + evt = p.get(1) + self.assertEquals(3, p.add_hashes(evt, md5='68b329da9893e34099c7d8ad5cb9c940', + sha1='adc83b19e793491b1c6ea0fd8b46cd9f32e592fc', + sha256='01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b', + filename='foobar.exe')) + self.assertEquals(3, p.add_hashes(evt, md5='68b329da9893e34099c7d8ad5cb9c940', + sha1='adc83b19e793491b1c6ea0fd8b46cd9f32e592fc', + sha256='01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b')) + p.av_detection_link(evt, 'https://foocorp.com') + p.add_detection_name(evt, 'WATERMELON') + p.add_filename(evt, 'foobar.exe') + p.add_regkey(evt, 'HKLM\\Software\\Microsoft\\Outlook\\Addins\\foobar') + p.add_regkey(evt, 'HKLM\\Software\\Microsoft\\Outlook\\Addins\\foobar', rvalue='foobar') + regkeys = { + 'HKLM\\Software\\Microsoft\\Outlook\\Addins\\foo': None, + 'HKLM\\Software\\Microsoft\\Outlook\\Addins\\bar': 'baz', + 'HKLM\\Software\\Microsoft\\Outlook\\Addins\\bae': 0, + } + self.assertEquals(3, p.add_regkeys(evt, regkeys)) + p.add_pattern(evt, '.*foobar.*', in_memory=True) + p.add_pattern(evt, '.*foobar.*', in_file=True) + self.assertRaises(pm.PyMISPError, p.add_pattern, evt, '.*foobar.*', in_memory=False, in_file=False) + p.add_pipe(evt, 'foo') + p.add_pipe(evt, '\\.\\pipe\\foo') + self.assertEquals(3, p.add_pipe(evt, ['foo', 'bar', 'baz'])) + self.assertEquals(3, p.add_pipe(evt, ['foo', 'bar', '\\.\\pipe\\baz'])) + p.add_mutex(evt, 'foo') + self.assertEquals(1, p.add_mutex(evt, '\\BaseNamedObjects\\foo')) + self.assertEquals(3, p.add_mutex(evt, ['foo', 'bar', 'baz'])) + self.assertEquals(3, p.add_mutex(evt, ['foo', 'bar', '\\BaseNamedObjects\\baz'])) + p.add_yara(evt, 'rule Foo {}') + self.assertEquals(2, p.add_yara(evt, ['rule Foo {}', 'rule Bar {}'])) + p.add_ipdst(evt, '1.2.3.4') + self.assertEquals(2, p.add_ipdst(evt, ['1.2.3.4', '5.6.7.8'])) + p.add_ipsrc(evt, '1.2.3.4') + self.assertEquals(2, p.add_ipsrc(evt, ['1.2.3.4', '5.6.7.8'])) + p.add_hostname(evt, 'a.foobar.com') + self.assertEquals(2, p.add_hostname(evt, ['a.foobar.com', 'a.foobaz.com'])) + p.add_domain(evt, 'foobar.com') + self.assertEquals(2, p.add_domain(evt, ['foobar.com', 'foobaz.com'])) + p.add_domain_ip(evt, 'foo.com', '1.2.3.4') + self.assertEquals(2, p.add_domain_ip(evt, 'foo.com', ['1.2.3.4', '5.6.7.8'])) + self.assertEquals(2, p.add_domains_ips(evt, {'foo.com': '1.2.3.4', 'bar.com': '4.5.6.7'})) + p.add_url(evt, 'https://example.com') + self.assertEquals(2, p.add_url(evt, ['https://example.com', 'http://foo.com'])) + p.add_useragent(evt, 'Mozilla') + self.assertEquals(2, p.add_useragent(evt, ['Mozilla', 'Godzilla'])) + p.add_traffic_pattern(evt, 'blabla') + p.add_snort(evt, 'blaba') + p.add_net_other(evt, 'blabla') + p.add_email_src(evt, 'foo@bar.com') + p.add_email_dst(evt, 'foo@bar.com') + p.add_email_subject(evt, 'you won the lottery') + p.add_email_attachment(evt, 'foo.doc') + p.add_target_email(evt, 'foo@bar.com') + p.add_target_user(evt, 'foo') + p.add_target_machine(evt, 'foobar') + p.add_target_org(evt, 'foobar') + p.add_target_location(evt, 'foobar') + p.add_target_external(evt, 'foobar') + p.add_threat_actor(evt, 'WATERMELON') + p.add_internal_link(evt, 'foobar') + p.add_internal_comment(evt, 'foobar') + p.add_internal_text(evt, 'foobar') + p.add_internal_other(evt, 'foobar') + if __name__ == '__main__': unittest.main() From 802fc0f20ebbabc655765d40918590e5f170c9f1 Mon Sep 17 00:00:00 2001 From: Nicolas Bareil Date: Thu, 1 Dec 2016 10:59:30 +0100 Subject: [PATCH 186/223] python3 does not like lambda (x,y) syntax --- pymisp/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/api.py b/pymisp/api.py index 55d8852..77acb0a 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -477,7 +477,7 @@ class PyMISP(object): return self.add_named_attribute(event, 'domain|ip', composed, category, to_ids, comment, distribution, proposal) def add_domains_ips(self, event, domain_ips, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): - composed = map(lambda (x,y): '%s|%s' % (x, y), domain_ips.items()) + composed = map(lambda x: '%s|%s' % (x[0], x[1]), domain_ips.items()) return self.add_named_attribute(event, 'domain|ip', composed, category, to_ids, comment, distribution, proposal) def add_url(self, event, url, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): From 59b7d199701abc61b28c7c084d2a9a337d115fcc Mon Sep 17 00:00:00 2001 From: Nicolas Bareil Date: Thu, 1 Dec 2016 14:26:59 +0100 Subject: [PATCH 187/223] map() is a generator in Python3 --- pymisp/api.py | 8 ++++---- tests/test_offline.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 77acb0a..6c3e391 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -444,7 +444,7 @@ class PyMISP(object): if not s.startswith('\\.\\pipe\\'): s = '\\.\\pipe\\{}'.format(s) return s - attributes = map(scrub, self._one_or_more(named_pipe)) + attributes = list(map(scrub, self._one_or_more(named_pipe))) return self.add_named_attribute(event, 'named pipe', attributes, category, to_ids, comment, distribution, proposal) def add_mutex(self, event, mutex, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False): @@ -452,7 +452,7 @@ class PyMISP(object): if not s.startswith('\\BaseNamedObjects\\'): s = '\\BaseNamedObjects\\{}'.format(s) return self - attributes = map(scrub, self._one_or_more(mutex)) + attributes = list(map(scrub, self._one_or_more(mutex))) return self.add_named_attribute(event, 'mutex', attributes, category, to_ids, comment, distribution, proposal) def add_yara(self, event, yara, category='Payload delivery', to_ids=False, comment=None, distribution=None, proposal=False): @@ -473,11 +473,11 @@ class PyMISP(object): return self.add_named_attribute(event, 'domain', domain, category, to_ids, comment, distribution, proposal) def add_domain_ip(self, event, domain, ip, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): - composed = map(lambda x: '%s|%s' % (domain, x), ip) + composed = list(map(lambda x: '%s|%s' % (domain, x), ip)) return self.add_named_attribute(event, 'domain|ip', composed, category, to_ids, comment, distribution, proposal) def add_domains_ips(self, event, domain_ips, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): - composed = map(lambda x: '%s|%s' % (x[0], x[1]), domain_ips.items()) + composed = list(map(lambda x: '%s|%s' % (x[0], x[1]), domain_ips.items())) return self.add_named_attribute(event, 'domain|ip', composed, category, to_ids, comment, distribution, proposal) def add_url(self, event, url, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False): diff --git a/tests/test_offline.py b/tests/test_offline.py index c4a8a1a..f32eeea 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -150,7 +150,7 @@ class TestOffline(unittest.TestCase): 'HKLM\\Software\\Microsoft\\Outlook\\Addins\\bar': 'baz', 'HKLM\\Software\\Microsoft\\Outlook\\Addins\\bae': 0, } - self.assertEquals(3, p.add_regkeys(evt, regkeys)) + self.assertEqual(3, p.add_regkeys(evt, regkeys)) p.add_pattern(evt, '.*foobar.*', in_memory=True) p.add_pattern(evt, '.*foobar.*', in_file=True) self.assertRaises(pm.PyMISPError, p.add_pattern, evt, '.*foobar.*', in_memory=False, in_file=False) From 31cf37fa020db665e47e03d0afc50009037ca3c4 Mon Sep 17 00:00:00 2001 From: morallo Date: Fri, 2 Dec 2016 16:52:00 +0100 Subject: [PATCH 188/223] Added test case for search_index by tag --- tests/search_index_result.json | 69 ++++++++++++++++++++++++++++++++++ tests/test_offline.py | 8 ++++ 2 files changed, 77 insertions(+) create mode 100644 tests/search_index_result.json diff --git a/tests/search_index_result.json b/tests/search_index_result.json new file mode 100644 index 0000000..bef46d0 --- /dev/null +++ b/tests/search_index_result.json @@ -0,0 +1,69 @@ +[ + { + "id": "3", + "org": "", + "date": "2016-12-01", + "info": "Another random Event", + "published": false, + "uuid": "5758ebf5-c898-48e6-9fe9-5665c0a83866", + "attribute_count": "2", + "analysis": "0", + "orgc": "", + "timestamp": "1465681801", + "distribution": "3", + "proposal_email_lock": false, + "locked": false, + "threat_level_id": "1", + "publish_timestamp": "0", + "sharing_group_id": "0", + "org_id": "1", + "orgc_id": "1", + "Org": { + "id": "1", + "name": "ORGNAME" + }, + "Orgc": { + "id": "1", + "name": "ORGNAME" + }, + "EventTag": [ + { + "id": "9760", + "event_id": "6028", + "tag_id": "4", + "Tag": { + "id": "4", + "name": "TLP:GREEN", + "colour": "#33822d", + "exportable": true + } + }, + { + "id": "9801", + "event_id": "3", + "tag_id": "1", + "Tag": { + "id": "1", + "name": "for_intelmq_processing", + "colour": "#00ad1c", + "exportable": true + } + }, + { + "id": "9803", + "event_id": "3", + "tag_id": "6", + "Tag": { + "id": "6", + "name": "ecsirt:malicious-code=\"ransomware\"", + "colour": "#005a5a", + "exportable": true + } + } + ], + "SharingGroup": { + "id": null, + "name": null + } + } +] diff --git a/tests/test_offline.py b/tests/test_offline.py index b4854af..162b641 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -29,6 +29,7 @@ class TestOffline(unittest.TestCase): self.auth_error_msg = {"name": "Authentication failed. Please make sure you pass the API key of an API enabled user along in the Authorization header.", "message": "Authentication failed. Please make sure you pass the API key of an API enabled user along in the Authorization header.", "url": "\/events\/1"} + self.search_index_result = json.load(open('tests/search_index_result.json', 'r')) def initURI(self, m): m.register_uri('GET', self.domain + 'events/1', json=self.auth_error_msg, status_code=403) @@ -40,6 +41,7 @@ class TestOffline(unittest.TestCase): m.register_uri('DELETE', self.domain + 'events/2', json={'message': 'Event deleted.'}) m.register_uri('DELETE', self.domain + 'events/3', json={'errors': ['Invalid event'], 'message': 'Invalid event', 'name': 'Invalid event', 'url': '/events/3'}) m.register_uri('DELETE', self.domain + 'attributes/2', json={'message': 'Attribute deleted.'}) + m.register_uri('GET', self.domain + 'events/index/searchtag:1', json=self.search_index_result) def test_getEvent(self, m): self.initURI(m) @@ -126,5 +128,11 @@ class TestOffline(unittest.TestCase): json.dumps(misp_event, cls=EncodeUpdate) json.dumps(misp_event, cls=EncodeFull) + def test_searchIndexByTag (self, m): + self.initURI(m) + pymisp = PyMISP(self.domain, self.key) + response = pymisp.search_index(tag="1") + self.assertEqual(response['response'],self.search_index_result) + if __name__ == '__main__': unittest.main() From a6a851f38d089e68b18980fd02a19941377ddea0 Mon Sep 17 00:00:00 2001 From: morallo Date: Fri, 2 Dec 2016 16:53:45 +0100 Subject: [PATCH 189/223] Solved warnings in tests when run under Python3 --- pymisp/mispevent.py | 6 ++++-- tests/test_offline.py | 18 ++++++++++++------ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 1fdd0cf..e59fb1b 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -216,8 +216,10 @@ class MISPEvent(object): def __init__(self, describe_types=None): self.ressources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') - self.json_schema = json.load(open(os.path.join(self.ressources_path, 'schema.json'), 'r')) - self.json_schema_lax = json.load(open(os.path.join(self.ressources_path, 'schema-lax.json'), 'r')) + with open(os.path.join(self.ressources_path, 'schema.json', 'r') as f: + self.json_schema = json.load(f) + with open(os.path.join(self.ressources_path, 'schema-lax.json'), 'r') as f: + self.json_schema_lax = json.load(f) if not describe_types: t = json.load(open(os.path.join(self.ressources_path, 'describeTypes.json'), 'r')) describe_types = t['result'] diff --git a/tests/test_offline.py b/tests/test_offline.py index 162b641..dc1a54a 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -21,15 +21,20 @@ class TestOffline(unittest.TestCase): self.maxDiff = None self.domain = 'http://misp.local/' self.key = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' - self.event = {'Event': json.load(open('tests/misp_event.json', 'r'))} - self.new_misp_event = {'Event': json.load(open('tests/new_misp_event.json', 'r'))} + with open('tests/misp_event.json', 'r') as f: + self.event = {'Event': json.load(f)} + with open('tests/new_misp_event.json', 'r') as f: + self.new_misp_event = {'Event': json.load(f)} self.ressources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), '../pymisp/data') - self.types = json.load(open(os.path.join(self.ressources_path, 'describeTypes.json'), 'r')) - self.sharing_groups = json.load(open('tests/sharing_groups.json', 'r')) + with open(os.path.join(self.ressources_path, 'describeTypes.json'), 'r') as f: + self.types = json.load(f) + with open('tests/sharing_groups.json', 'r') as f: + self.sharing_groups = json.load(f) self.auth_error_msg = {"name": "Authentication failed. Please make sure you pass the API key of an API enabled user along in the Authorization header.", "message": "Authentication failed. Please make sure you pass the API key of an API enabled user along in the Authorization header.", "url": "\/events\/1"} - self.search_index_result = json.load(open('tests/search_index_result.json', 'r')) + with open('tests/search_index_result.json', 'r') as f: + self.search_index_result = json.load(f) def initURI(self, m): m.register_uri('GET', self.domain + 'events/1', json=self.auth_error_msg, status_code=403) @@ -124,7 +129,8 @@ class TestOffline(unittest.TestCase): self.initURI(m) pymisp = PyMISP(self.domain, self.key) misp_event = MISPEvent(pymisp.describe_types) - misp_event.load(open('tests/57c4445b-c548-4654-af0b-4be3950d210f.json', 'r').read()) + with open('tests/57c4445b-c548-4654-af0b-4be3950d210f.json', 'r') as f: + misp_event.load(f.read()) json.dumps(misp_event, cls=EncodeUpdate) json.dumps(misp_event, cls=EncodeFull) From d677bc58a9b6127abc534b5f6a4836dadfa5de1f Mon Sep 17 00:00:00 2001 From: morallo Date: Fri, 2 Dec 2016 17:43:33 +0100 Subject: [PATCH 190/223] Fixed missing parenthesis --- pymisp/mispevent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index e59fb1b..4659101 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -216,7 +216,7 @@ class MISPEvent(object): def __init__(self, describe_types=None): self.ressources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') - with open(os.path.join(self.ressources_path, 'schema.json', 'r') as f: + with open(os.path.join(self.ressources_path, 'schema.json', 'r')) as f: self.json_schema = json.load(f) with open(os.path.join(self.ressources_path, 'schema-lax.json'), 'r') as f: self.json_schema_lax = json.load(f) From c40eaf935f1d19723109b71df00307b66078efb6 Mon Sep 17 00:00:00 2001 From: morallo Date: Fri, 2 Dec 2016 17:43:33 +0100 Subject: [PATCH 191/223] Fixed synthax error --- pymisp/mispevent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index e59fb1b..611141b 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -216,7 +216,7 @@ class MISPEvent(object): def __init__(self, describe_types=None): self.ressources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') - with open(os.path.join(self.ressources_path, 'schema.json', 'r') as f: + with open(os.path.join(self.ressources_path, 'schema.json'),'r') as f: self.json_schema = json.load(f) with open(os.path.join(self.ressources_path, 'schema-lax.json'), 'r') as f: self.json_schema_lax = json.load(f) From f96c1b57484933d1cc36d2b09bc9d6b0ec2363a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sat, 3 Dec 2016 18:37:13 +0100 Subject: [PATCH 192/223] Reorganize json dumps --- pymisp/mispevent.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index e6a4c62..3682ef7 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -163,8 +163,6 @@ class MISPAttribute(object): to_return = {'type': self.type, 'category': self.category, 'to_ids': self.to_ids, 'distribution': self.distribution, 'value': self.value, 'comment': self.comment} - if self.sharing_group_id: - to_return['sharing_group_id'] = self.sharing_group_id if self.sig: to_return['sig'] = self.sig to_return = _int_to_str(to_return) @@ -172,6 +170,8 @@ class MISPAttribute(object): def _json_full(self): to_return = self._json() + if self.sharing_group_id: + to_return['sharing_group_id'] = self.sharing_group_id if self.id: to_return['id'] = self.id if self.uuid: @@ -430,21 +430,15 @@ class MISPEvent(object): 'threat_level_id': self.threat_level_id, 'analysis': self.analysis, 'Attribute': []} if self.sig: - to_return['sig'] = self.sig + to_return['Event']['sig'] = self.sig if self.global_sig: - to_return['global_sig'] = self.global_sig - if self.id: - to_return['Event']['id'] = self.id - if self.orgc_id: - to_return['Event']['orgc_id'] = self.orgc_id - if self.org_id: - to_return['Event']['org_id'] = self.org_id + to_return['Event']['global_sig'] = self.global_sig if self.uuid: to_return['Event']['uuid'] = self.uuid - if self.sharing_group_id: - to_return['Event']['sharing_group_id'] = self.sharing_group_id if self.Tag: to_return['Event']['Tag'] = self.Tag + if self.Orgc: + to_return['Event']['Orgc'] = self.Orgc to_return['Event'] = _int_to_str(to_return['Event']) if self.attributes: to_return['Event']['Attribute'] = [a._json() for a in self.attributes] @@ -453,6 +447,14 @@ class MISPEvent(object): def _json_full(self): to_return = self._json() + if self.id: + to_return['Event']['id'] = self.id + if self.orgc_id: + to_return['Event']['orgc_id'] = self.orgc_id + if self.org_id: + to_return['Event']['org_id'] = self.org_id + if self.sharing_group_id: + to_return['Event']['sharing_group_id'] = self.sharing_group_id if self.locked is not None: to_return['Event']['locked'] = self.locked if self.attribute_count is not None: @@ -461,8 +463,6 @@ class MISPEvent(object): to_return['Event']['RelatedEvent'] = self.RelatedEvent if self.Org: to_return['Event']['Org'] = self.Org - if self.Orgc: - to_return['Event']['Orgc'] = self.Orgc if self.ShadowAttribute: to_return['Event']['ShadowAttribute'] = self.ShadowAttribute if self.proposal_email_lock is not None: From 13dbaef5b7983ef0d6ee760659bb68cb76cd4ee6 Mon Sep 17 00:00:00 2001 From: Iglocska Date: Wed, 7 Dec 2016 10:01:55 +0100 Subject: [PATCH 193/223] Added galaxyies to the test --- tests/test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test.py b/tests/test.py index cef2b5f..a227655 100755 --- a/tests/test.py +++ b/tests/test.py @@ -47,7 +47,8 @@ class TestBasic(unittest.TestCase): u'distribution': u'0', u'Attribute': [], u'proposal_email_lock': False, u'Org': {u'name': u'ORGNAME'}, u'Orgc': {u'name': u'ORGNAME'}, - u'threat_level_id': u'1'}} + u'threat_level_id': u'1'}, + u'Galaxy': []} print(event) self.assertEqual(event, to_check, 'Failed at creating a new Event') return int(event_id) @@ -62,6 +63,7 @@ class TestBasic(unittest.TestCase): u'ShadowAttribute': [], u'published': False, u'distribution': u'0', u'Org': {u'name': u'ORGNAME'}, u'Orgc': {u'name': u'ORGNAME'}, + u'Galaxy': [], u'Attribute': [ {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a', @@ -85,6 +87,7 @@ class TestBasic(unittest.TestCase): u'ShadowAttribute': [], u'published': True, u'distribution': u'0', u'Org': {u'name': u'ORGNAME'}, u'Orgc': {u'name': u'ORGNAME'}, + u'Galaxy': [], u'Attribute': [ {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a', From 5bc4db789b22c694713965ac47736934f85cf2bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 7 Dec 2016 10:54:22 +0100 Subject: [PATCH 194/223] Fix typo in add_mutex --- pymisp/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/api.py b/pymisp/api.py index 6c3e391..61a54ef 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -451,7 +451,7 @@ class PyMISP(object): def scrub(s): if not s.startswith('\\BaseNamedObjects\\'): s = '\\BaseNamedObjects\\{}'.format(s) - return self + return s attributes = list(map(scrub, self._one_or_more(mutex))) return self.add_named_attribute(event, 'mutex', attributes, category, to_ids, comment, distribution, proposal) From db182ce28677056a0752c2166cc3e7708fc8f315 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 7 Dec 2016 11:32:14 +0100 Subject: [PATCH 195/223] Ignore order in event --- tests/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test.py b/tests/test.py index a227655..3dcf7b9 100755 --- a/tests/test.py +++ b/tests/test.py @@ -50,7 +50,7 @@ class TestBasic(unittest.TestCase): u'threat_level_id': u'1'}, u'Galaxy': []} print(event) - self.assertEqual(event, to_check, 'Failed at creating a new Event') + self.assertCountEqual(event, to_check, 'Failed at creating a new Event') return int(event_id) def add_hashes(self, eventid): From 8cec9377c1c4e2ea756282e9bfee7701200c94b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 7 Dec 2016 14:35:49 +0100 Subject: [PATCH 196/223] Fix tests --- tests/test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test.py b/tests/test.py index 3dcf7b9..d1a2ac9 100755 --- a/tests/test.py +++ b/tests/test.py @@ -47,10 +47,10 @@ class TestBasic(unittest.TestCase): u'distribution': u'0', u'Attribute': [], u'proposal_email_lock': False, u'Org': {u'name': u'ORGNAME'}, u'Orgc': {u'name': u'ORGNAME'}, - u'threat_level_id': u'1'}, - u'Galaxy': []} + u'Galaxy' : [], + u'threat_level_id': u'1'}} print(event) - self.assertCountEqual(event, to_check, 'Failed at creating a new Event') + self.assertEqual(event, to_check, 'Failed at creating a new Event') return int(event_id) def add_hashes(self, eventid): From 81faa507f4f03b2ccce132437232fc7216f15122 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 7 Dec 2016 18:37:35 +0100 Subject: [PATCH 197/223] Add support for data field (malware-sample) --- pymisp/mispevent.py | 46 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 3682ef7..fabf6d9 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -8,6 +8,10 @@ from json import JSONEncoder import os import warnings import base64 +from io import BytesIO +from zipfile import ZipFile +import hashlib + try: from dateutil.parser import parse except ImportError: @@ -63,6 +67,8 @@ class MISPAttribute(object): self.distribution = 5 # other possible values + self.data = None + self.encrypt = False self.id = None self.uuid = None self.timestamp = None @@ -142,6 +148,9 @@ class MISPAttribute(object): raise NewAttributeError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 5'.format(self.distribution)) # other possible values + if kwargs.get('data'): + self.data = kwargs['data'] + self._load_data() if kwargs.get('id'): self.id = int(kwargs['id']) if kwargs.get('uuid'): @@ -159,12 +168,47 @@ class MISPAttribute(object): if kwargs.get('sig'): self.sig = kwargs['sig'] + def _prepare_new_malware_sample(self): + if '|' in self.value: + # Get the filename, ignore the md5, because humans. + self.malware_filename, md5 = self.value.split('|') + else: + # Assuming the user only passed the filename + self.malware_filename = self.value + m = hashlib.md5() + m.update(self.data.getvalue()) + md5 = m.hexdigest() + self.value = '{}|{}'.format(self.malware_filename, md5) + self.malware_binary = self.data + self.encrypt = True + + def _load_data(self): + if not isinstance(self.data, BytesIO): + self.data = BytesIO(base64.b64decode(self.data)) + if self.type == 'malware-sample': + try: + with ZipFile(self.data) as f: + for name in f.namelist(): + if name.endswith('.txt'): + with f.open(name, pwd=b'infected') as unpacked: + self.malware_filename = unpacked.read().decode() + else: + with f.open(name, pwd=b'infected') as unpacked: + self.malware_binary = BytesIO(unpacked.read()) + except: + # not a encrypted zip file, assuming it is a new malware sample + self._prepare_new_malware_sample() + def _json(self): to_return = {'type': self.type, 'category': self.category, 'to_ids': self.to_ids, 'distribution': self.distribution, 'value': self.value, 'comment': self.comment} if self.sig: to_return['sig'] = self.sig + if self.data: + to_return['data'] = base64.b64encode(self.data.getvalue()).decode() + if self.encrypt: + to_return['entrypt'] = self.encrypt to_return = _int_to_str(to_return) return to_return @@ -216,7 +260,7 @@ class MISPEvent(object): def __init__(self, describe_types=None): self.ressources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') - with open(os.path.join(self.ressources_path, 'schema.json'),'r') as f: + with open(os.path.join(self.ressources_path, 'schema.json'), 'r') as f: self.json_schema = json.load(f) with open(os.path.join(self.ressources_path, 'schema-lax.json'), 'r') as f: self.json_schema_lax = json.load(f) From 83c9ce92187e6582535b93dfb24dcd8bda7cb97e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 9 Dec 2016 11:42:07 +0100 Subject: [PATCH 198/223] Allow to change the to_ids flag of an attribute --- pymisp/api.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pymisp/api.py b/pymisp/api.py index 61a54ef..f240229 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -655,6 +655,17 @@ class PyMISP(object): session = self.__prepare_session() return self.__query_proposal(session, 'discard', proposal_id) + # ############################## + # ###### Attribute update ###### + # ############################## + + def change_toids(self, attribute_uuid, to_ids): + if to_ids not in [0, 1]: + raise Exception('to_ids can only be 0 or 1') + query = {"to_ids": to_ids} + session = self.__prepare_session() + return self.__query(session, 'edit/{}'.format(attribute_uuid), query, controller='attributes') + # ############################## # ######## REST Search ######### # ############################## @@ -665,7 +676,6 @@ class PyMISP(object): if controller not in ['events', 'attributes']: raise Exception('Invalid controller. Can only be {}'.format(', '.join(['events', 'attributes']))) url = urljoin(self.root_url, '{}/{}'.format(controller, path.lstrip('/'))) - query = {'request': query} if self.debug: print('URL: ', url) print('Query: ', query) From a2ec3bf5518c3d26bc805e432a7c249fa1b6c0f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 9 Dec 2016 17:21:59 +0100 Subject: [PATCH 199/223] Bump to v2.4.56 --- pymisp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 4aad873..e168ce7 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.4.54' +__version__ = '2.4.56' from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey from .api import PyMISP From 0f2206a70030006b6f5f7a04cf7e3ef891f86d48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 9 Dec 2016 17:32:03 +0100 Subject: [PATCH 200/223] Add basic support for Galaxy --- pymisp/mispevent.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index fabf6d9..5885047 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -305,6 +305,7 @@ class MISPEvent(object): self.ShadowAttribute = [] self.RelatedEvent = [] self.Tag = [] + self.Galaxy = None def _serialize(self): return '{date}{threat_level_id}{info}{uuid}{analysis}{timestamp}'.format( @@ -460,6 +461,8 @@ class MISPEvent(object): self.ShadowAttribute = kwargs['ShadowAttribute'] if kwargs.get('RelatedEvent'): self.RelatedEvent = kwargs['RelatedEvent'] + if kwargs.get('Galaxy'): + self.Galaxy = kwargs['Galaxy'] if kwargs.get('Tag'): self.Tag = kwargs['Tag'] if kwargs.get('sig'): @@ -483,6 +486,8 @@ class MISPEvent(object): to_return['Event']['Tag'] = self.Tag if self.Orgc: to_return['Event']['Orgc'] = self.Orgc + if self.Galaxy: + to_return['Event']['Galaxy'] = self.Galaxy to_return['Event'] = _int_to_str(to_return['Event']) if self.attributes: to_return['Event']['Attribute'] = [a._json() for a in self.attributes] From d4489d9c346eb1ff82abb0f6c0c7bca20878ce57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 14 Dec 2016 15:17:33 +0100 Subject: [PATCH 201/223] Fix sharing group distribution level. Fix https://github.com/MISP/MISP/issues/1761 --- pymisp/mispevent.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 5885047..74d72dc 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -144,8 +144,8 @@ class MISPAttribute(object): self.comment = kwargs['comment'] if kwargs.get('distribution'): self.distribution = int(kwargs['distribution']) - if self.distribution not in [0, 1, 2, 3, 5]: - raise NewAttributeError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 5'.format(self.distribution)) + if self.distribution not in [0, 1, 2, 3, 4, 5]: + raise NewAttributeError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 4, 5'.format(self.distribution)) # other possible values if kwargs.get('data'): @@ -205,6 +205,8 @@ class MISPAttribute(object): 'comment': self.comment} if self.sig: to_return['sig'] = self.sig + if self.sharing_group_id: + to_return['sharing_group_id'] = self.sharing_group_id if self.data: to_return['data'] = base64.b64encode(self.data.getvalue()).decode() if self.encrypt: @@ -214,8 +216,6 @@ class MISPAttribute(object): def _json_full(self): to_return = self._json() - if self.sharing_group_id: - to_return['sharing_group_id'] = self.sharing_group_id if self.id: to_return['id'] = self.id if self.uuid: @@ -412,8 +412,8 @@ class MISPEvent(object): # Default values for a valid event to send to a MISP instance if kwargs.get('distribution') is not None: self.distribution = int(kwargs['distribution']) - if self.distribution not in [0, 1, 2, 3]: - raise NewEventError('{} is invalid, the distribution has to be in 0, 1, 2, 3'.format(self.distribution)) + if self.distribution not in [0, 1, 2, 3, 4]: + raise NewEventError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 4'.format(self.distribution)) if kwargs.get('threat_level_id') is not None: self.threat_level_id = int(kwargs['threat_level_id']) if self.threat_level_id not in [1, 2, 3, 4]: @@ -488,6 +488,8 @@ class MISPEvent(object): to_return['Event']['Orgc'] = self.Orgc if self.Galaxy: to_return['Event']['Galaxy'] = self.Galaxy + if self.sharing_group_id: + to_return['Event']['sharing_group_id'] = self.sharing_group_id to_return['Event'] = _int_to_str(to_return['Event']) if self.attributes: to_return['Event']['Attribute'] = [a._json() for a in self.attributes] @@ -502,8 +504,6 @@ class MISPEvent(object): to_return['Event']['orgc_id'] = self.orgc_id if self.org_id: to_return['Event']['org_id'] = self.org_id - if self.sharing_group_id: - to_return['Event']['sharing_group_id'] = self.sharing_group_id if self.locked is not None: to_return['Event']['locked'] = self.locked if self.attribute_count is not None: From 4f230c9299ad9d2d1c851148c629b61a94f3f117 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 14 Dec 2016 15:42:43 +0100 Subject: [PATCH 202/223] Add warning of PyMISP and MISP version don't match. --- pymisp/api.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pymisp/api.py b/pymisp/api.py index f240229..4693cb2 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -100,7 +100,19 @@ class PyMISP(object): try: # Make sure the MISP instance is working and the URL is valid - self.get_version() + response = self.get_version() + misp_version = response['version'].split('.') + pymisp_version = __version__.split('.') + for a, b in zip(misp_version, pymisp_version): + if a == b: + continue + elif a < b: + warnings.warn("Remote MISP instance (v{}) older than PyMISP (v{}). You should update your MISP instance, or install an older PyMISP version.".format(response['version'], __version__)) + else: # a > b + # NOTE: That can happen and should not be blocking + warnings.warn("Remote MISP instance (v{}) newer than PyMISP (v{}). Please check if a newer version of PyMISP is available.".format(response['version'], __version__)) + continue + except Exception as e: raise PyMISPError('Unable to connect to MISP ({}). Please make sure the API key and the URL are correct (http/https is required): {}'.format(self.root_url, e)) From 0511bd44867f5f7f74cb54d61b0df7d99b719118 Mon Sep 17 00:00:00 2001 From: Georges Bossert Date: Tue, 27 Dec 2016 15:13:24 +0100 Subject: [PATCH 203/223] Fix typo in comments of 'search_index' method definition --- pymisp/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/api.py b/pymisp/api.py index 4693cb2..33d9b65 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -704,7 +704,7 @@ class PyMISP(object): :param eventid: Evend ID(s) | str or list :param tag: Tag(s) | str or list :param datefrom: First date, in format YYYY-MM-DD - :param datefrom: Last date, in format YYYY-MM-DD + :param dateto: Last date, in format YYYY-MM-DD :param eventinfo: Event info(s) to match | str or list :param threatlevel: Threat level(s) (1,2,3,4) | str or list :param distribution: Distribution level(s) (0,1,2,3) | str or list From 585ca9cd08388a86a2d6093426c374746ebbdafd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 2 Jan 2017 16:53:23 +0100 Subject: [PATCH 204/223] Allow to mark an attribute as deleted in a MISPEvent Related to #33 --- pymisp/mispevent.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 74d72dc..8e6825a 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -534,6 +534,16 @@ class MISPEvent(object): def unpublish(self): self.published = False + def delete_attribute(self, attribute_id): + found = False + for a in self.attributes: + if a.id == attribute_id or a.uuid == attribute_id: + a.deleted = True + found = True + break + if not found: + raise Exception('No attribute with UUID/ID {} found.'.format(attribute_id)) + def add_attribute(self, type, value, **kwargs): attribute = MISPAttribute(self.describe_types) attribute.set_all_values(type=type, value=value, **kwargs) From bfb9fd5db3015a421b9d04f0dc54324f04d508d4 Mon Sep 17 00:00:00 2001 From: Tristan METAYER Date: Wed, 4 Jan 2017 11:23:18 +0100 Subject: [PATCH 205/223] Add uuid serch with pymisp --- pymisp/api.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/pymisp/api.py b/pymisp/api.py index f12b668..f10485f 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -368,6 +368,19 @@ class PyMISP(object): return self._check_response(response) + def _valid_uuid(self,uuid): + """ + Test valid uuid générate by CakePhp + + CakeText::uuid follow RFC 4122 + - the third group must start with a 4, + - the fourth group must start with 8, 9, a or b. + + :param uuid: A UUID to validate + """ + regex = re.compile('^[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}\Z', re.I) + match = regex.match(uuid) + return bool(match) # ##### File attributes ##### def _send_attributes(self, event, attributes, proposal=False): @@ -732,7 +745,7 @@ class PyMISP(object): def search(self, values=None, not_values=None, type_attribute=None, category=None, org=None, tags=None, not_tags=None, date_from=None, - date_to=None, last=None): + date_to=None, last=None, uuid=None): """ Search via the Rest API @@ -746,6 +759,7 @@ class PyMISP(object): :param date_from: First date :param date_to: Last date :param last: Last updated events (for example 5d or 12h or 30m) + :param uuid: A uuid valideted """ val = self.__prepare_rest_search(values, not_values).replace('/', '|') @@ -773,6 +787,11 @@ class PyMISP(object): query['to'] = date_to if last is not None: query['last'] = last + if uuid is not None: + if self._valid_uuid(uuid): + query['uuid'] = uuid + else: + return {'error': 'You must enter a valid uuid.'} session = self.__prepare_session('json') return self.__query(session, 'restSearch/download', query) From 879f60f36855bee0b436ba7d9d3c2d9c53edef86 Mon Sep 17 00:00:00 2001 From: Tristan METAYER Date: Wed, 4 Jan 2017 17:02:13 +0100 Subject: [PATCH 206/223] refere to FloatingGhost comment --- pymisp/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index f10485f..d85efd7 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -376,7 +376,7 @@ class PyMISP(object): - the third group must start with a 4, - the fourth group must start with 8, 9, a or b. - :param uuid: A UUID to validate + :param uuid: an uuid """ regex = re.compile('^[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}\Z', re.I) match = regex.match(uuid) @@ -759,7 +759,7 @@ class PyMISP(object): :param date_from: First date :param date_to: Last date :param last: Last updated events (for example 5d or 12h or 30m) - :param uuid: A uuid valideted + :param uuid: a valid uuid """ val = self.__prepare_rest_search(values, not_values).replace('/', '|') From 5478778a38c04ff30f5fdd7bc500a7dd8b9e39f3 Mon Sep 17 00:00:00 2001 From: Tristan METAYER Date: Wed, 4 Jan 2017 17:04:36 +0100 Subject: [PATCH 207/223] idem --- pymisp/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/api.py b/pymisp/api.py index d85efd7..d3f21f6 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -370,7 +370,7 @@ class PyMISP(object): def _valid_uuid(self,uuid): """ - Test valid uuid générate by CakePhp + Test if uuid is valid CakeText::uuid follow RFC 4122 - the third group must start with a 4, From b2752bd8d5a1d048c8b576dd86de5575ca42e011 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 6 Jan 2017 16:24:39 -0500 Subject: [PATCH 208/223] Load RelatedEvent as MISPEvent. Fix #76 --- pymisp/mispevent.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 8e6825a..02ae66c 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -95,6 +95,9 @@ class MISPAttribute(object): signed, _ = c.sign(to_sign, mode=mode.DETACH) self.sig = base64.b64encode(signed).decode() + def delete(self): + self.deleted = True + def verify(self, gpg_uid): if not has_pyme: raise Exception('pyme is required, please install: pip install --pre pyme3. You will also need libgpg-error-dev and libgpgme11-dev.') @@ -460,7 +463,11 @@ class MISPEvent(object): if kwargs.get('ShadowAttribute'): self.ShadowAttribute = kwargs['ShadowAttribute'] if kwargs.get('RelatedEvent'): - self.RelatedEvent = kwargs['RelatedEvent'] + self.RelatedEvent = [] + for rel_event in kwargs['RelatedEvent']: + sub_event = MISPEvent() + sub_event.load(rel_event) + self.RelatedEvent.append(sub_event) if kwargs.get('Galaxy'): self.Galaxy = kwargs['Galaxy'] if kwargs.get('Tag'): @@ -509,7 +516,9 @@ class MISPEvent(object): if self.attribute_count is not None: to_return['Event']['attribute_count'] = self.attribute_count if self.RelatedEvent: - to_return['Event']['RelatedEvent'] = self.RelatedEvent + to_return['Event']['RelatedEvent'] = [] + for rel_event in self.RelatedEvent: + to_return['Event']['RelatedEvent'].append(rel_event._json_full()) if self.Org: to_return['Event']['Org'] = self.Org if self.ShadowAttribute: @@ -538,7 +547,7 @@ class MISPEvent(object): found = False for a in self.attributes: if a.id == attribute_id or a.uuid == attribute_id: - a.deleted = True + a.delete() found = True break if not found: From 2f967268d1b249fe4ce5784b46e9392c2d2d483e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 6 Jan 2017 21:03:02 -0500 Subject: [PATCH 209/223] Add new key in online test --- tests/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test.py b/tests/test.py index d1a2ac9..dfd93aa 100755 --- a/tests/test.py +++ b/tests/test.py @@ -42,12 +42,12 @@ class TestBasic(unittest.TestCase): event = self.misp.new_event(0, 1, 0, "This is a test") event_id = self._clean_event(event) to_check = {u'Event': {u'info': u'This is a test', u'locked': False, - u'attribute_count': None, u'analysis': u'0', + u'attribute_count': None, 'disable_correlation': False, u'analysis': u'0', u'ShadowAttribute': [], u'published': False, u'distribution': u'0', u'Attribute': [], u'proposal_email_lock': False, u'Org': {u'name': u'ORGNAME'}, u'Orgc': {u'name': u'ORGNAME'}, - u'Galaxy' : [], + u'Galaxy': [], u'threat_level_id': u'1'}} print(event) self.assertEqual(event, to_check, 'Failed at creating a new Event') From 60c02cb5e9dd00bb9c6fed9a093a2d0b44511d14 Mon Sep 17 00:00:00 2001 From: cgi Date: Mon, 9 Jan 2017 16:19:20 +0100 Subject: [PATCH 210/223] + separate function change_sharing_group using update_event --- pymisp/api.py | 21 +++++++++++++++------ pymisp/mispevent.py | 2 ++ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index f846a8e..c0227f3 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -44,7 +44,7 @@ class distributions(object): this_community = 1 connected_communities = 2 all_communities = 3 - + sharing_group = 4 class threat_level(object): """Enumeration of the available threat levels.""" @@ -309,10 +309,10 @@ class PyMISP(object): # ######### Event handling (Json only) ######### # ############################################## - def _prepare_full_event(self, distribution, threat_level_id, analysis, info, date=None, published=False, orgc_id=None, org_id=None): + def _prepare_full_event(self, distribution, threat_level_id, analysis, info, date=None, published=False, orgc_id=None, org_id=None, sharing_group_id=None): misp_event = MISPEvent(self.describe_types) misp_event.set_all_values(info=info, distribution=distribution, threat_level_id=threat_level_id, - analysis=analysis, date=date, orgc_id=orgc_id, org_id=org_id) + analysis=analysis, date=date, orgc_id=orgc_id, org_id=org_id, sharing_group_id=sharing_group_id) if published: misp_event.publish() return misp_event @@ -353,8 +353,16 @@ class PyMISP(object): e.threat_level_id = threat_level_id return self.update_event(event['Event']['id'], json.dumps(e, cls=EncodeUpdate)) - def new_event(self, distribution=None, threat_level_id=None, analysis=None, info=None, date=None, published=False, orgc_id=None, org_id=None): - misp_event = self._prepare_full_event(distribution, threat_level_id, analysis, info, date, published, orgc_id, org_id) + def change_sharing_group(self, event, sharing_group_id): + e = MISPEvent(self.describe_types) + e.load(event) + e.distribution = 4 # Needs to be 'Sharing group' + e.sharing_group_id = sharing_group_id + return self.update_event(event['Event']['id'], json.dumps(e, cls=EncodeUpdate)) + + + def new_event(self, distribution=None, threat_level_id=None, analysis=None, info=None, date=None, published=False, orgc_id=None, org_id=None, sharing_group_id=None): + misp_event = self._prepare_full_event(distribution, threat_level_id, analysis, info, date, published, orgc_id, org_id, sharing_group_id) return self.add_event(json.dumps(misp_event, cls=EncodeUpdate)) def add_tag(self, event, tag): @@ -1046,10 +1054,11 @@ class PyMISP(object): def get_sharing_groups(self): session = self.__prepare_session() - url = urljoin(self.root_url, 'sharing_groups/index.json') + url = urljoin(self.root_url, 'sharing_groups') response = session.get(url) return self._check_response(response)['response'][0] + # ############## Users ################## def _set_user_parameters(self, email, org_id, role_id, password, external_auth_required, diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 8e6825a..03e3396 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -512,6 +512,8 @@ class MISPEvent(object): to_return['Event']['RelatedEvent'] = self.RelatedEvent if self.Org: to_return['Event']['Org'] = self.Org + if self.sharing_group_id: + to_return['Event']['sharing_group_id'] = self.sharing_group_id if self.ShadowAttribute: to_return['Event']['ShadowAttribute'] = self.ShadowAttribute if self.proposal_email_lock is not None: From 56a18275bb3df476ce54c9837168ddc6333b2777 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 11 Jan 2017 14:29:20 -0500 Subject: [PATCH 211/223] Fix get sharing groups Fix #79 --- pymisp/api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index f846a8e..9729bc6 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -370,13 +370,13 @@ class PyMISP(object): return self._check_response(response) def _valid_uuid(self,uuid): - """ + """ Test if uuid is valid CakeText::uuid follow RFC 4122 - the third group must start with a 4, - the fourth group must start with 8, 9, a or b. - + :param uuid: an uuid """ regex = re.compile('^[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}\Z', re.I) @@ -1048,7 +1048,7 @@ class PyMISP(object): session = self.__prepare_session() url = urljoin(self.root_url, 'sharing_groups/index.json') response = session.get(url) - return self._check_response(response)['response'][0] + return self._check_response(response)['response'] # ############## Users ################## From 15fb54b032825b15ec536d3f839ce1e6b39b1d77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 11 Jan 2017 14:50:58 -0500 Subject: [PATCH 212/223] Update tests --- tests/test_offline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_offline.py b/tests/test_offline.py index 87ce654..6867fd1 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -103,7 +103,7 @@ class TestOffline(unittest.TestCase): self.initURI(m) pymisp = PyMISP(self.domain, self.key) sharing_groups = pymisp.get_sharing_groups() - self.assertEqual(sharing_groups, self.sharing_groups['response'][0]) + self.assertEqual(sharing_groups[0], self.sharing_groups['response'][0]) def test_auth_error(self, m): self.initURI(m) From cfb8572ab155c3cdb3b4528d4f9ef6c5c5347e1c Mon Sep 17 00:00:00 2001 From: Hannah Ward Date: Fri, 13 Jan 2017 13:15:53 +0000 Subject: [PATCH 213/223] new: Added ability to add attachments to events --- .gitignore | 1 + pymisp/api.py | 37 +++++++++++++++++++++++++++++++++---- tests/test_offline.py | 2 +- 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 4fb001d..39eab06 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +*.swp *.pem *.pyc examples/keys.py diff --git a/pymisp/api.py b/pymisp/api.py index 9729bc6..e186622 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -317,10 +317,10 @@ class PyMISP(object): misp_event.publish() return misp_event - def _prepare_full_attribute(self, category, type_value, value, to_ids, comment=None, distribution=5): + def _prepare_full_attribute(self, category, type_value, value, to_ids, comment=None, distribution=5, **kwargs): misp_attribute = MISPAttribute(self.describe_types) misp_attribute.set_all_values(type=type_value, value=value, category=category, - to_ids=to_ids, comment=comment, distribution=distribution) + to_ids=to_ids, comment=comment, distribution=distribution, **kwargs) return misp_attribute def _one_or_more(self, value): @@ -394,10 +394,10 @@ class PyMISP(object): response = self.update_event(event['Event']['id'], json.dumps(e, cls=EncodeUpdate)) return response - def add_named_attribute(self, event, type_value, value, category=None, to_ids=False, comment=None, distribution=None, proposal=False): + def add_named_attribute(self, event, type_value, value, category=None, to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): attributes = [] for value in self._one_or_more(value): - attributes.append(self._prepare_full_attribute(category, type_value, value, to_ids, comment, distribution)) + attributes.append(self._prepare_full_attribute(category, type_value, value, to_ids, comment, distribution, **kwargs)) return self._send_attributes(event, attributes, proposal) def add_hashes(self, event, category='Artifacts dropped', filename=None, md5=None, sha1=None, sha256=None, ssdeep=None, comment=None, to_ids=True, distribution=None, proposal=False): @@ -432,6 +432,35 @@ class PyMISP(object): def add_filename(self, event, filename, category='Artifacts dropped', to_ids=False, comment=None, distribution=None, proposal=False): return self.add_named_attribute(event, 'filename', filename, category, to_ids, comment, distribution, proposal) + def add_attachment(self, event, filename, attachment=None, category='Artifacts dropped', to_ids=False, comment=None, distribution=None, proposal=False): + """Add an attachment to the MISP event + + :param event: The event to add an attachment to + :param filename: The name you want to store the file under + :param attachment: Either a file handle or a path to a file - will be uploaded + """ + + if hasattr(attachment, "read"): + # It's a file handle - we can read it + fileData = attachment.read() + + elif isinstance(attachment, str): + # It can either be the b64 encoded data or a file path + if os.path.exists(attachment): + # It's a path! + with open(attachment, "r") as f: + fileData = f.read() + else: + # We have to assume it's the actual data + fileData = attachment + + # by now we have a string for the file + # we just need to b64 encode it and send it on its way + # also, just decode it to utf-8 to avoid the b'string' format + encodedData = base64.b64encode(fileData.encode("utf-8")).decode("utf-8") + + # Send it on its way + return self.add_named_attribute(event, 'attachment', filename, category, to_ids, comment, distribution, proposal, data=encodedData) def add_regkey(self, event, regkey, rvalue=None, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False): if rvalue: type_value = 'regkey|value' diff --git a/tests/test_offline.py b/tests/test_offline.py index 6867fd1..f4ffd25 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -218,6 +218,6 @@ class TestOffline(unittest.TestCase): p.add_internal_comment(evt, 'foobar') p.add_internal_text(evt, 'foobar') p.add_internal_other(evt, 'foobar') - + p.add_attachment(evt, "testFile", "Attacment added!") if __name__ == '__main__': unittest.main() From 78cef069637c132427e144eda869ae492e6ab3ef Mon Sep 17 00:00:00 2001 From: Hannah Ward Date: Mon, 16 Jan 2017 09:52:35 +0000 Subject: [PATCH 214/223] new: Added ability to disable correlation on attributes --- pymisp/mispevent.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 02ae66c..a2a0f92 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -77,6 +77,7 @@ class MISPAttribute(object): self.sig = None self.SharingGroup = [] self.ShadowAttribute = [] + self.disable_correlation = False def _serialize(self): return '{type}{category}{to_ids}{uuid}{timestamp}{comment}{deleted}{value}'.format( @@ -171,6 +172,9 @@ class MISPAttribute(object): if kwargs.get('sig'): self.sig = kwargs['sig'] + # If the user wants to disable correlation, let them. Defaults to False. + self.disable_correlation = kwargs.get("disable_correlation", False) + def _prepare_new_malware_sample(self): if '|' in self.value: # Get the filename, ignore the md5, because humans. @@ -205,7 +209,7 @@ class MISPAttribute(object): def _json(self): to_return = {'type': self.type, 'category': self.category, 'to_ids': self.to_ids, 'distribution': self.distribution, 'value': self.value, - 'comment': self.comment} + 'comment': self.comment, 'disable_correlation': self.disable_correlation} if self.sig: to_return['sig'] = self.sig if self.sharing_group_id: From 7c8cde0afd14223f6443142f7a6140d829146d21 Mon Sep 17 00:00:00 2001 From: Hannah Ward Date: Mon, 16 Jan 2017 14:27:44 +0000 Subject: [PATCH 215/223] chg: Updated api.py docstrings to comply with PEP257 --- pymisp/api.py | 177 ++++++++++++++++++++------------------------------ 1 file changed, 72 insertions(+), 105 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index e186622..1a769cd 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -62,19 +62,18 @@ class analysis(object): class PyMISP(object): - """ - Python API for MISP + """Python API for MISP - :param url: URL of the MISP instance you want to connect to - :param key: API key of the user you want to use - :param ssl: can be True or False (to check ot not the validity - of the certificate. Or a CA_BUNDLE in case of self - signed certiifcate (the concatenation of all the - *.crt of the chain) - :param out_type: Type of object (json) NOTE: XML output isn't supported anymore, keeping the flag for compatibility reasons. - :param debug: print all the messages received from the server - :param proxies: Proxy dict as describes here: http://docs.python-requests.org/en/master/user/advanced/#proxies - :param cert: Client certificate, as described there: http://docs.python-requests.org/en/master/user/advanced/#ssl-cert-verification + :param url: URL of the MISP instance you want to connect to + :param key: API key of the user you want to use + :param ssl: can be True or False (to check ot not the validity + of the certificate. Or a CA_BUNDLE in case of self + signed certiifcate (the concatenation of all the + *.crt of the chain) + :param out_type: Type of object (json) NOTE: XML output isn't supported anymore, keeping the flag for compatibility reasons. + :param debug: print all the messages received from the server + :param proxies: Proxy dict as describes here: http://docs.python-requests.org/en/master/user/advanced/#proxies + :param cert: Client certificate, as described there: http://docs.python-requests.org/en/master/user/advanced/#ssl-cert-verification """ # So it can may be accessed from the misp object. @@ -136,9 +135,8 @@ class PyMISP(object): self.sane_default = self.describe_types['sane_defaults'] def __prepare_session(self, output='json'): - """ - Prepare the headers of the session - """ + """Prepare the headers of the session""" + if not HAVE_REQUESTS: raise MissingDependency('Missing dependency, install requests (`pip install requests`)') session = requests.Session() @@ -218,10 +216,9 @@ class PyMISP(object): # ################################################ def get_index(self, filters=None): - """ - Return the index. + """Return the index. - Warning, there's a limit on the number of results + Warning, there's a limit on the number of results """ session = self.__prepare_session() url = urljoin(self.root_url, 'events/index') @@ -233,10 +230,9 @@ class PyMISP(object): return self._check_response(response) def get_event(self, event_id): - """ - Get an event + """Get an event - :param event_id: Event id to get + :param event_id: Event id to get """ session = self.__prepare_session() url = urljoin(self.root_url, 'events/{}'.format(event_id)) @@ -244,9 +240,7 @@ class PyMISP(object): return self._check_response(response) def get_stix_event(self, event_id=None, with_attachments=False, from_date=False, to_date=False, tags=False): - """ - Get an event/events in STIX format - """ + """Get an event/events in STIX format""" if tags: if isinstance(tags, list): tags = "&&".join(tags) @@ -260,10 +254,9 @@ class PyMISP(object): return self._check_response(response) def add_event(self, event): - """ - Add a new event - - :param event: Event as JSON object / string or XML to add + """Add a new event + + :param event: Event as JSON object / string or XML to add """ session = self.__prepare_session() url = urljoin(self.root_url, 'events') @@ -274,11 +267,10 @@ class PyMISP(object): return self._check_response(response) def update_event(self, event_id, event): - """ - Update an event + """Update an event - :param event_id: Event id to update - :param event: Event as JSON object / string or XML to add + :param event_id: Event id to update + :param event: Event as JSON object / string or XML to add """ session = self.__prepare_session() url = urljoin(self.root_url, 'events/{}'.format(event_id)) @@ -289,10 +281,9 @@ class PyMISP(object): return self._check_response(response) def delete_event(self, event_id): - """ - Delete an event + """Delete an event - :param event_id: Event id to delete + :param event_id: Event id to delete """ session = self.__prepare_session() url = urljoin(self.root_url, 'events/{}'.format(event_id)) @@ -370,14 +361,12 @@ class PyMISP(object): return self._check_response(response) def _valid_uuid(self,uuid): - """ - Test if uuid is valid + """Test if uuid is valid + Will test against CakeText's RFC 4122, i.e + "the third group must start with a 4, + and the fourth group must start with 8, 9, a or b." - CakeText::uuid follow RFC 4122 - - the third group must start with a 4, - - the fourth group must start with 8, 9, a or b. - - :param uuid: an uuid + :param uuid: an uuid """ regex = re.compile('^[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}\Z', re.I) match = regex.match(uuid) @@ -739,20 +728,18 @@ class PyMISP(object): def search_index(self, published=None, eventid=None, tag=None, datefrom=None, dateto=None, eventinfo=None, threatlevel=None, distribution=None, analysis=None, attribute=None, org=None): - """ - Search only at the index level. Use ! infront of value as NOT, default OR - - :param published: Published (0,1) - :param eventid: Evend ID(s) | str or list - :param tag: Tag(s) | str or list - :param datefrom: First date, in format YYYY-MM-DD - :param dateto: Last date, in format YYYY-MM-DD - :param eventinfo: Event info(s) to match | str or list - :param threatlevel: Threat level(s) (1,2,3,4) | str or list - :param distribution: Distribution level(s) (0,1,2,3) | str or list - :param analysis: Analysis level(s) (0,1,2) | str or list - :param org: Organisation(s) | str or list + """Search only at the index level. Use ! infront of value as NOT, default OR + :param published: Published (0,1) + :param eventid: Evend ID(s) | str or list + :param tag: Tag(s) | str or list + :param datefrom: First date, in format YYYY-MM-DD + :param dateto: Last date, in format YYYY-MM-DD + :param eventinfo: Event info(s) to match | str or list + :param threatlevel: Threat level(s) (1,2,3,4) | str or list + :param distribution: Distribution level(s) (0,1,2,3) | str or list + :param analysis: Analysis level(s) (0,1,2) | str or list + :param org: Organisation(s) | str or list """ allowed = {'published': published, 'eventid': eventid, 'tag': tag, 'Dateto': dateto, 'Datefrom': datefrom, 'eventinfo': eventinfo, 'threatlevel': threatlevel, @@ -787,11 +774,10 @@ class PyMISP(object): return self.__query(session, 'restSearch/download', query) def __prepare_rest_search(self, values, not_values): - """ - Prepare a search, generate the chain processed by the server + """Prepare a search, generate the chain processed by the server - :param values: Values to search - :param not_values: Values that should not be in the response + :param values: Values to search + :param not_values: Values that should not be in the response """ to_return = '' if values is not None: @@ -813,21 +799,20 @@ class PyMISP(object): def search(self, values=None, not_values=None, type_attribute=None, category=None, org=None, tags=None, not_tags=None, date_from=None, date_to=None, last=None, metadata=None, uuid=None, controller='events'): - """ - Search via the Rest API + """Search via the Rest API - :param values: values to search for - :param not_values: values *not* to search for - :param type_attribute: Type of attribute - :param category: Category to search - :param org: Org reporting the event - :param tags: Tags to search for - :param not_tags: Tags *not* to search for - :param date_from: First date - :param date_to: Last date - :param last: Last updated events (for example 5d or 12h or 30m) - :param metadata: return onlymetadata if True - :param uuid: a valid uuid + :param values: values to search for + :param not_values: values *not* to search for + :param type_attribute: Type of attribute + :param category: Category to search + :param org: Org reporting the event + :param tags: Tags to search for + :param not_tags: Tags *not* to search for + :param date_from: First date + :param date_to: Last date + :param last: Last updated events (for example 5d or 12h or 30m) + :param metadata: return onlymetadata if True + :param uuid: a valid uuid """ val = self.__prepare_rest_search(values, not_values) tag = self.__prepare_rest_search(tags, not_tags) @@ -865,12 +850,10 @@ class PyMISP(object): session = self.__prepare_session() return self.__query(session, 'restSearch/download', query, controller) - def get_attachement(self, event_id): - """ - Get attachement of an event (not sample) + def get_attachment(self, event_id): + """Get attachement of an event (not sample) - :param event_id: Event id from where the attachements will - be fetched + :param event_id: Event id from where the attachements will be fetched """ attach = urljoin(self.root_url, 'attributes/downloadAttachment/download/{}'.format(event_id)) session = self.__prepare_session() @@ -918,29 +901,25 @@ class PyMISP(object): return True, details def download_last(self, last): - """ - Download the last updated events. + """Download the last updated events. - :param last: can be defined in days, hours, minutes (for example 5d or 12h or 30m) + :param last: can be defined in days, hours, minutes (for example 5d or 12h or 30m) """ return self.search(last=last) # ############## Suricata ############### def download_all_suricata(self): - """ - Download all suricata rules events. - """ + """Download all suricata rules events.""" suricata_rules = urljoin(self.root_url, 'events/nids/suricata/download') session = self.__prepare_session('rules') response = session.get(suricata_rules) return response def download_suricata_rule_event(self, event_id): - """ - Download one suricata rule event. + """Download one suricata rule event. - :param event_id: ID of the event to download (same as get) + :param event_id: ID of the event to download (same as get) """ template = urljoin(self.root_url, 'events/nids/suricata/download/{}'.format(event_id)) session = self.__prepare_session('rules') @@ -972,15 +951,11 @@ class PyMISP(object): # ########## Version ########## def get_api_version(self): - """ - Returns the current version of PyMISP installed on the system - """ + """Returns the current version of PyMISP installed on the system""" return {'version': __version__} def get_api_version_master(self): - """ - Get the most recent version of PyMISP from github - """ + """Get the most recent version of PyMISP from github""" r = requests.get('https://raw.githubusercontent.com/MISP/PyMISP/master/pymisp/__init__.py') if r.status_code == 200: version = re.findall("__version__ = '(.*)'", r.text) @@ -989,18 +964,14 @@ class PyMISP(object): return {'error': 'Impossible to retrieve the version of the master branch.'} def get_version(self): - """ - Returns the version of the instance. - """ + """Returns the version of the instance.""" session = self.__prepare_session() url = urljoin(self.root_url, 'servers/getVersion.json') response = session.get(url) return self._check_response(response) def get_version_master(self): - """ - Get the most recent version from github - """ + """Get the most recent version from github""" r = requests.get('https://raw.githubusercontent.com/MISP/MISP/2.4/VERSION.json') if r.status_code == 200: master_version = json.loads(r.text) @@ -1020,9 +991,7 @@ class PyMISP(object): # ############## Statistics ################## def get_attributes_statistics(self, context='type', percentage=None): - """ - Get attributes statistics from the MISP instance - """ + """Get attributes statistics from the MISP instance""" session = self.__prepare_session() if (context != 'category'): context = 'type' @@ -1034,9 +1003,7 @@ class PyMISP(object): return self._check_response(response) def get_tags_statistics(self, percentage=None, name_sort=None): - """ - Get tags statistics from the MISP instance - """ + """Get tags statistics from the MISP instance""" session = self.__prepare_session() if percentage is not None: percentage = 'true' From 748136f3f19ac2dc8cc6188813aaf1c987e212ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 16 Jan 2017 14:41:32 -0500 Subject: [PATCH 216/223] Allow to update an event by UUID, syntax fixes. --- pymisp/api.py | 16 ++++++++++------ pymisp/mispevent.py | 3 ++- pymisp/tools/stix.py | 2 +- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 1a769cd..a2cccb8 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -255,7 +255,7 @@ class PyMISP(object): def add_event(self, event): """Add a new event - + :param event: Event as JSON object / string or XML to add """ session = self.__prepare_session() @@ -327,7 +327,10 @@ class PyMISP(object): return self.get_stix_event(**kwargs) def update(self, event): - eid = event['Event']['id'] + if event['Event'].get('uuid'): + eid = event['Event']['uuid'] + else: + eid = event['Event']['id'] return self.update_event(eid, event) def publish(self, event): @@ -360,7 +363,7 @@ class PyMISP(object): response = session.post(urljoin(self.root_url, 'events/removeTag'), data=json.dumps(to_post)) return self._check_response(response) - def _valid_uuid(self,uuid): + def _valid_uuid(self, uuid): """Test if uuid is valid Will test against CakeText's RFC 4122, i.e "the third group must start with a 4, @@ -422,13 +425,13 @@ class PyMISP(object): return self.add_named_attribute(event, 'filename', filename, category, to_ids, comment, distribution, proposal) def add_attachment(self, event, filename, attachment=None, category='Artifacts dropped', to_ids=False, comment=None, distribution=None, proposal=False): - """Add an attachment to the MISP event + """Add an attachment to the MISP event :param event: The event to add an attachment to :param filename: The name you want to store the file under :param attachment: Either a file handle or a path to a file - will be uploaded """ - + if hasattr(attachment, "read"): # It's a file handle - we can read it fileData = attachment.read() @@ -447,9 +450,10 @@ class PyMISP(object): # we just need to b64 encode it and send it on its way # also, just decode it to utf-8 to avoid the b'string' format encodedData = base64.b64encode(fileData.encode("utf-8")).decode("utf-8") - + # Send it on its way return self.add_named_attribute(event, 'attachment', filename, category, to_ids, comment, distribution, proposal, data=encodedData) + def add_regkey(self, event, regkey, rvalue=None, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False): if rvalue: type_value = 'regkey|value' diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index a2a0f92..6441179 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -78,6 +78,7 @@ class MISPAttribute(object): self.SharingGroup = [] self.ShadowAttribute = [] self.disable_correlation = False + self.RelatedAttribute = [] def _serialize(self): return '{type}{category}{to_ids}{uuid}{timestamp}{comment}{deleted}{value}'.format( @@ -172,7 +173,7 @@ class MISPAttribute(object): if kwargs.get('sig'): self.sig = kwargs['sig'] - # If the user wants to disable correlation, let them. Defaults to False. + # If the user wants to disable correlation, let them. Defaults to False. self.disable_correlation = kwargs.get("disable_correlation", False) def _prepare_new_malware_sample(self): diff --git a/pymisp/tools/stix.py b/pymisp/tools/stix.py index b6463c8..c3a81fb 100644 --- a/pymisp/tools/stix.py +++ b/pymisp/tools/stix.py @@ -3,7 +3,7 @@ try: from misp_stix_converter.converters.buildMISPAttribute import buildEvent - from misp_stix_converter.converters import convert + from misp_stix_converter.converters import convert from misp_stix_converter.converters.convert import MISPtoSTIX has_misp_stix_converter = True except ImportError: From 619538ced782cbdcc4a4d47097ef128b9bb5e003 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 16 Jan 2017 14:47:43 -0500 Subject: [PATCH 217/223] Fix travis --- pymisp/api.py | 41 +++++++++++++++++++++++------------------ tests/test_offline.py | 12 ++++++------ 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 4324158..e85e9de 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -46,6 +46,7 @@ class distributions(object): all_communities = 3 sharing_group = 4 + class threat_level(object): """Enumeration of the available threat levels.""" high = 1 @@ -320,6 +321,14 @@ class PyMISP(object): # ########## Helpers ########## + def _make_mispevent(self, event): + if not isinstance(event, MISPEvent): + e = MISPEvent(self.describe_types) + e.load(event) + else: + e = event + return e + def get(self, eid): return self.get_event(eid) @@ -327,33 +336,30 @@ class PyMISP(object): return self.get_stix_event(**kwargs) def update(self, event): - if event['Event'].get('uuid'): - eid = event['Event']['uuid'] + e = self._make_mispevent(event) + if e.uuid: + eid = e.uuid else: - eid = event['Event']['id'] - return self.update_event(eid, event) + eid = e.id + return self.update_event(eid, json.dumps(e, cls=EncodeUpdate)) def publish(self, event): - if event['Event']['published']: + e = self._make_mispevent(event) + if e.published: return {'error': 'Already published'} - e = MISPEvent(self.describe_types) - e.load(event) e.publish() - return self.update_event(event['Event']['id'], json.dumps(e, cls=EncodeUpdate)) + return self.update(event) def change_threat_level(self, event, threat_level_id): - e = MISPEvent(self.describe_types) - e.load(event) + e = self._make_mispevent(event) e.threat_level_id = threat_level_id - return self.update_event(event['Event']['id'], json.dumps(e, cls=EncodeUpdate)) + return self.update(event) def change_sharing_group(self, event, sharing_group_id): - e = MISPEvent(self.describe_types) - e.load(event) + e = self._make_mispevent(event) e.distribution = 4 # Needs to be 'Sharing group' e.sharing_group_id = sharing_group_id - return self.update_event(event['Event']['id'], json.dumps(e, cls=EncodeUpdate)) - + return self.update(event) def new_event(self, distribution=None, threat_level_id=None, analysis=None, info=None, date=None, published=False, orgc_id=None, org_id=None, sharing_group_id=None): misp_event = self._prepare_full_event(distribution, threat_level_id, analysis, info, date, published, orgc_id, org_id, sharing_group_id) @@ -391,7 +397,7 @@ class PyMISP(object): e = MISPEvent(self.describe_types) e.load(event) e.attributes += attributes - response = self.update_event(event['Event']['id'], json.dumps(e, cls=EncodeUpdate)) + response = self.update(event) return response def add_named_attribute(self, event, type_value, value, category=None, to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): @@ -1054,11 +1060,10 @@ class PyMISP(object): def get_sharing_groups(self): session = self.__prepare_session() - url = urljoin(self.root_url, 'sharing_groups') + url = urljoin(self.root_url, 'sharing_groups.json') response = session.get(url) return self._check_response(response)['response'] - # ############## Users ################## def _set_user_parameters(self, email, org_id, role_id, password, external_auth_required, diff --git a/tests/test_offline.py b/tests/test_offline.py index f4ffd25..298485c 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -38,11 +38,11 @@ class TestOffline(unittest.TestCase): def initURI(self, m): m.register_uri('GET', self.domain + 'events/1', json=self.auth_error_msg, status_code=403) - m.register_uri('GET', self.domain + 'servers/getVersion.json', json={"version": "2.4.50"}) - m.register_uri('GET', self.domain + 'sharing_groups/index.json', json=self.sharing_groups) + m.register_uri('GET', self.domain + 'servers/getVersion.json', json={"version": "2.4.56"}) + m.register_uri('GET', self.domain + 'sharing_groups.json', json=self.sharing_groups) m.register_uri('GET', self.domain + 'attributes/describeTypes.json', json=self.types) m.register_uri('GET', self.domain + 'events/2', json=self.event) - m.register_uri('POST', self.domain + 'events/2', json=self.event) + m.register_uri('POST', self.domain + 'events/5758ebf5-c898-48e6-9fe9-5665c0a83866', json=self.event) m.register_uri('DELETE', self.domain + 'events/2', json={'message': 'Event deleted.'}) m.register_uri('DELETE', self.domain + 'events/3', json={'errors': ['Invalid event'], 'message': 'Invalid event', 'name': 'Invalid event', 'url': '/events/3'}) m.register_uri('DELETE', self.domain + 'attributes/2', json={'message': 'Attribute deleted.'}) @@ -60,8 +60,8 @@ class TestOffline(unittest.TestCase): def test_updateEvent(self, m): self.initURI(m) pymisp = PyMISP(self.domain, self.key) - e0 = pymisp.update_event(2, json.dumps(self.event)) - e1 = pymisp.update_event(2, self.event) + e0 = pymisp.update_event('5758ebf5-c898-48e6-9fe9-5665c0a83866', json.dumps(self.event)) + e1 = pymisp.update_event('5758ebf5-c898-48e6-9fe9-5665c0a83866', self.event) self.assertEqual(e0, e1) e2 = pymisp.update(e0) self.assertEqual(e1, e2) @@ -97,7 +97,7 @@ class TestOffline(unittest.TestCase): api_version = pymisp.get_api_version() self.assertEqual(api_version, {'version': pm.__version__}) server_version = pymisp.get_version() - self.assertEqual(server_version, {"version": "2.4.50"}) + self.assertEqual(server_version, {"version": "2.4.56"}) def test_getSharingGroups(self, m): self.initURI(m) From 2bcc4163adbe825aff5c9189c402da8c5e3da5c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 17 Jan 2017 18:20:24 -0500 Subject: [PATCH 218/223] Add support for attribute level tagging --- pymisp/api.py | 17 +++++++++++++---- pymisp/mispevent.py | 5 +++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index e85e9de..cb0179b 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -365,15 +365,24 @@ class PyMISP(object): misp_event = self._prepare_full_event(distribution, threat_level_id, analysis, info, date, published, orgc_id, org_id, sharing_group_id) return self.add_event(json.dumps(misp_event, cls=EncodeUpdate)) - def add_tag(self, event, tag): + def add_tag(self, event, tag, attribute=False): + # FIXME: this is dirty, this function needs to be deprecated with something tagging a UUID session = self.__prepare_session() - to_post = {'request': {'Event': {'id': event['Event']['id'], 'tag': tag}}} + if attribute: + to_post = {'request': {'Attribute': {'id': event['id'], 'tag': tag}}} + else: + to_post = {'request': {'Event': {'id': event['id'], 'tag': tag}}} response = session.post(urljoin(self.root_url, 'events/addTag'), data=json.dumps(to_post)) return self._check_response(response) - def remove_tag(self, event, tag): + def remove_tag(self, event, tag, attribute=False): + # FIXME: this is dirty, this function needs to be deprecated with something removing the tag to a UUID session = self.__prepare_session() - to_post = {'request': {'Event': {'id': event['Event']['id'], 'tag': tag}}} + if attribute: + to_post = {'request': {'Attribute': {'id': event['id'], 'tag': tag}}} + pass + else: + to_post = {'request': {'Event': {'id': event['Event']['id'], 'tag': tag}}} response = session.post(urljoin(self.root_url, 'events/removeTag'), data=json.dumps(to_post)) return self._check_response(response) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 66c172a..e8c2d6c 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -79,6 +79,7 @@ class MISPAttribute(object): self.ShadowAttribute = [] self.disable_correlation = False self.RelatedAttribute = [] + self.Tag = [] def _serialize(self): return '{type}{category}{to_ids}{uuid}{timestamp}{comment}{deleted}{value}'.format( @@ -172,6 +173,8 @@ class MISPAttribute(object): self.ShadowAttribute = kwargs['ShadowAttribute'] if kwargs.get('sig'): self.sig = kwargs['sig'] + if kwargs.get('Tag'): + self.Tag = kwargs['Tag'] # If the user wants to disable correlation, let them. Defaults to False. self.disable_correlation = kwargs.get("disable_correlation", False) @@ -215,6 +218,8 @@ class MISPAttribute(object): to_return['sig'] = self.sig if self.sharing_group_id: to_return['sharing_group_id'] = self.sharing_group_id + if self.Tag: + to_return['Event']['Tag'] = self.Tag if self.data: to_return['data'] = base64.b64encode(self.data.getvalue()).decode() if self.encrypt: From f593ce69f87a2f1c3c1752f0a9c714d418069853 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 17 Jan 2017 19:03:38 -0500 Subject: [PATCH 219/223] Fix last commit --- pymisp/api.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index cb0179b..c58d31a 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -370,9 +370,11 @@ class PyMISP(object): session = self.__prepare_session() if attribute: to_post = {'request': {'Attribute': {'id': event['id'], 'tag': tag}}} + path = 'attributes/addTag' else: to_post = {'request': {'Event': {'id': event['id'], 'tag': tag}}} - response = session.post(urljoin(self.root_url, 'events/addTag'), data=json.dumps(to_post)) + path = 'events/addTag' + response = session.post(urljoin(self.root_url, path), data=json.dumps(to_post)) return self._check_response(response) def remove_tag(self, event, tag, attribute=False): @@ -380,10 +382,11 @@ class PyMISP(object): session = self.__prepare_session() if attribute: to_post = {'request': {'Attribute': {'id': event['id'], 'tag': tag}}} - pass + path = 'attributes/addTag' else: to_post = {'request': {'Event': {'id': event['Event']['id'], 'tag': tag}}} - response = session.post(urljoin(self.root_url, 'events/removeTag'), data=json.dumps(to_post)) + path = 'events/addTag' + response = session.post(urljoin(self.root_url, path), data=json.dumps(to_post)) return self._check_response(response) def _valid_uuid(self, uuid): From 351157b8f1627fe11b3464ab1954b5d542a66011 Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Wed, 18 Jan 2017 09:33:35 +0100 Subject: [PATCH 220/223] Minor documentation clarification --- pymisp/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/api.py b/pymisp/api.py index c58d31a..8b86bde 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1012,7 +1012,7 @@ class PyMISP(object): # ############## Export Attributes in text #################################### def get_all_attributes_txt(self, type_attr): - + """Get all attributes from a specific type as plain text. Only published and IDS flagged attributes are exported.""" session = self.__prepare_session('txt') url = urljoin(self.root_url, 'attributes/text/download/%s' % type_attr) response = session.get(url) From 10b95c778f38c36da87f7202598640086b817c15 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Wed, 18 Jan 2017 17:27:56 +0100 Subject: [PATCH 221/223] Doc link updated. Fix #39 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ed28248..204a434 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ python last.py -l 10 ## Documentation -[PyMISP API documentation is available](http://www.circl.lu/assets/files/PyMISP.pdf). +[PyMISP API documentation is available](https://media.readthedocs.org/pdf/pymisp/master/pymisp.pdf). Documentation can be generated with epydoc: From b26c021b0e47831b1b1612430cb010892a2c2769 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 19 Jan 2017 19:07:24 -0500 Subject: [PATCH 222/223] Fix typo --- pymisp/mispevent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index e8c2d6c..db52ced 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -219,7 +219,7 @@ class MISPAttribute(object): if self.sharing_group_id: to_return['sharing_group_id'] = self.sharing_group_id if self.Tag: - to_return['Event']['Tag'] = self.Tag + to_return['Tag'] = self.Tag if self.data: to_return['data'] = base64.b64encode(self.data.getvalue()).decode() if self.encrypt: From 3241e415b5cb166fffb14dcc1ac3beb7bde8d883 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 24 Jan 2017 07:15:38 +0100 Subject: [PATCH 223/223] Add options to restsearch calls. Related to: https://github.com/MISP/MISP/commit/8c63e6f3d54a262bc4bf6f77138c058287be5826 --- pymisp/api.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 8b86bde..572af35 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -757,7 +757,7 @@ class PyMISP(object): def search_index(self, published=None, eventid=None, tag=None, datefrom=None, dateto=None, eventinfo=None, threatlevel=None, distribution=None, - analysis=None, attribute=None, org=None): + analysis=None, attribute=None, org=None, to_ids=False, deleted=False): """Search only at the index level. Use ! infront of value as NOT, default OR :param published: Published (0,1) @@ -770,14 +770,25 @@ class PyMISP(object): :param distribution: Distribution level(s) (0,1,2,3) | str or list :param analysis: Analysis level(s) (0,1,2) | str or list :param org: Organisation(s) | str or list + :param to_ids: + - false (default): include all attributes, no matter the to_ids flag + - true: include only to_ids attributes + - "exclude": exclude attributes marked to_ids + :param deleted: + - false (default): only include non deleted attributes + - true: include deleted attributes + - "only": ONLY include deleted attributes """ allowed = {'published': published, 'eventid': eventid, 'tag': tag, 'Dateto': dateto, 'Datefrom': datefrom, 'eventinfo': eventinfo, 'threatlevel': threatlevel, 'distribution': distribution, 'analysis': analysis, 'attribute': attribute, - 'org': org} + 'org': org, 'to_ids': to_ids, 'deleted': deleted} rule_levels = {'distribution': ["0", "1", "2", "3", "!0", "!1", "!2", "!3"], 'threatlevel': ["1", "2", "3", "4", "!1", "!2", "!3", "!4"], - 'analysis': ["0", "1", "2", "!0", "!1", "!2"]} + 'analysis': ["0", "1", "2", "!0", "!1", "!2"], + 'to_ids': ['True', 'False', 'exclude'], + 'deleted': ['True', 'False', 'only'], + } buildup_url = "events/index" for rule in allowed.keys():