From c5f8cc88a2cb22b8e5801fa3958c2fa630560c7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 18 Oct 2017 08:30:35 +0200 Subject: [PATCH 01/36] chg: Bump misp-objects and describeTypes --- pymisp/data/describeTypes.json | 3 --- pymisp/data/misp-objects | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/pymisp/data/describeTypes.json b/pymisp/data/describeTypes.json index 6435855..5561114 100644 --- a/pymisp/data/describeTypes.json +++ b/pymisp/data/describeTypes.json @@ -805,8 +805,6 @@ "vulnerability", "x509-fingerprint-sha1", "other", - "ip-dst|port", - "ip-src|port", "hostname|port", "email-dst-display-name", "email-src-display-name", @@ -1023,7 +1021,6 @@ "text", "attachment", "comment", - "text", "other", "hex" ], diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index bc5795d..a5d2f71 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit bc5795dc189bcf9cd001527656fe10e6ed10dc5f +Subproject commit a5d2f71fef1552b6957fc74b4c2a8650355becae From c49008a636737965ee2f730c6938b982e1c269dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 18 Oct 2017 08:36:19 +0200 Subject: [PATCH 02/36] fix: Missing default category Fix #119 --- pymisp/data/describeTypes.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/data/describeTypes.json b/pymisp/data/describeTypes.json index 5561114..d5ee01c 100644 --- a/pymisp/data/describeTypes.json +++ b/pymisp/data/describeTypes.json @@ -438,7 +438,7 @@ "to_ids": 0 }, "email-message-id": { - "default_category": "", + "default_category": "Payload delivery", "to_ids": 0 }, "github-username": { From cae7f635eafe51c91925929d95532c5cf70a0bbf Mon Sep 17 00:00:00 2001 From: Andras Iklody Date: Wed, 18 Oct 2017 16:27:54 +0200 Subject: [PATCH 03/36] Update openioc.py --- pymisp/tools/openioc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pymisp/tools/openioc.py b/pymisp/tools/openioc.py index 2405028..ea0c880 100755 --- a/pymisp/tools/openioc.py +++ b/pymisp/tools/openioc.py @@ -11,7 +11,6 @@ except ImportError: has_bs4 = False iocMispMapping = { - # ~ @Link https://wiki.ops.fr/doku.php/manuels:misp:event-guidelines 'CookieHistoryItem/HostName': {'type': 'hostname', 'comment': 'CookieHistory.'}, 'DriverItem/DriverName': {'category': 'Artifacts dropped', 'type': 'other', 'comment': 'DriverName.'}, From 94e3419c39e29c3549892abc511f53e1f045118a Mon Sep 17 00:00:00 2001 From: garanews Date: Fri, 20 Oct 2017 09:55:46 +0200 Subject: [PATCH 04/36] Created add_generic_object.py usage: add_generic_object.py [-h] -e EVENT -t TYPE -d DICT Examples: python3 add_generic_object.py -e 1683 -t email -d '{"subject":"The Pink Letter", "to":"jon@snow.org"}' python3 add_generic_object.py -e 2343 -t person -d '{"first-name":"Daenerys", "last-name":"Targaryen", "place-of-birth":"Dragonstone"}' python3 add_generic_object.py -e 3596 -t "domain|ip" -d '{"domain":"stormborn.org", "ip":"50.63.202.33"}' --- examples/add_generic_object.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 examples/add_generic_object.py diff --git a/examples/add_generic_object.py b/examples/add_generic_object.py new file mode 100644 index 0000000..308a1a3 --- /dev/null +++ b/examples/add_generic_object.py @@ -0,0 +1,33 @@ +import json +from pymisp import PyMISP +from pymisp.tools.abstractgenerator import AbstractMISPObjectGenerator +from keys import misp_url, misp_key, misp_verifycert +import argparse + +class GenericObject(AbstractMISPObjectGenerator): + def __init__(self, type, data_dict): + super(GenericObject, self).__init__(type) + self.__data = data_dict + self.generate_attributes() + + def generate_attributes(self): + for key, value in self.__data.items(): + self.add_attribute(key, value=value) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Create a MISP Object selectable by type starting from a dictionary') + parser.add_argument("-e", "--event", required=True, help="Event ID to update") + parser.add_argument("-t", "--type", required=True, help="Type of the generic object") + parser.add_argument("-d", "--dict", required=True, help="Dict ") + args = parser.parse_args() + + pymisp = PyMISP(misp_url, misp_key, misp_verifycert) + try: + template_id = [x['ObjectTemplate']['id'] for x in pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == args.type][0] + except IndexError: + valid_types = ", ".join([x['ObjectTemplate']['name'] for x in pymisp.get_object_templates_list()]) + print ("Template for type %s not found! Valid types are: %s" % (args.type, valid_types)) + exit() + + misp_object = GenericObject(args.type.replace("|", "-"), json.loads(args.dict)) + r = pymisp.add_object(args.event, template_id, misp_object) From e2c400aac301c1a984cfb623eee82db107902076 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 20 Oct 2017 10:21:10 +0200 Subject: [PATCH 05/36] fix: Improve dependencies listing Partial fix for #110 --- .travis.yml | 4 +--- setup.py | 3 +++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 391d973..e932855 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,10 +21,8 @@ install: - pip install coveralls - pip install codecov - pip install requests-mock - - pip install https://github.com/lief-project/packages/raw/lief-master-latest/pylief-0.7.0.dev.zip - pip install git+https://github.com/kbandla/pydeep.git - - pip install python-magic - - pip install . + - pip install .[fileobjects,neo,openioc] - pushd tests - git clone https://github.com/viper-framework/viper-test-files.git - popd diff --git a/setup.py b/setup.py index 5b5d273..2d9dfee 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,9 @@ setup( ], test_suite="tests.test_offline", install_requires=['six', 'requests', 'python-dateutil', 'jsonschema'], + extras_require={'fileobjects': ['lief>=0.8', 'python-magic'], + 'neo': ['py2neo'], + 'openioc': ['beautifulsoup4']}, tests_require=[ 'jsonschema', 'python-dateutil', From f6febb5842d6a98c59a02b8968f404a805383c96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 20 Oct 2017 10:59:37 +0200 Subject: [PATCH 06/36] fix: min required version of setuptools --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2d9dfee..64b5e1a 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ setup( 'Topic :: Internet', ], test_suite="tests.test_offline", - install_requires=['six', 'requests', 'python-dateutil', 'jsonschema'], + install_requires=['six', 'requests', 'python-dateutil', 'jsonschema', 'setuptools>=36.4'], extras_require={'fileobjects': ['lief>=0.8', 'python-magic'], 'neo': ['py2neo'], 'openioc': ['beautifulsoup4']}, From 279533d7a85d1afcd88366f212efcac58b640965 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 20 Oct 2017 11:28:57 +0200 Subject: [PATCH 07/36] fix: Fix travis build --- .travis.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index e932855..e6a5ba7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,10 +17,8 @@ python: - "3.6-dev" install: - - pip install -U nose - - pip install coveralls - - pip install codecov - - pip install requests-mock + - pip install -U nose pip setuptools + - pip install coveralls codecov requests-mock - pip install git+https://github.com/kbandla/pydeep.git - pip install .[fileobjects,neo,openioc] - pushd tests From 45684cfd5f703957dd86057d9185b9abe36b0c17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sat, 21 Oct 2017 20:22:38 +0200 Subject: [PATCH 08/36] chg: Update comments --- pymisp/api.py | 106 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 97 insertions(+), 9 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 00e8b56..83105c6 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -147,7 +147,7 @@ class PyMISP(object): self.sane_default = self.describe_types['sane_defaults'] def __prepare_session(self, output='json', async_implemented=False): - """Prepare the headers of the session""" + """Prepare the session headers""" if not HAVE_REQUESTS: raise MissingDependency('Missing dependency, install requests (`pip install requests`)') @@ -170,6 +170,10 @@ class PyMISP(object): # ##################### def flatten_error_messages(self, response): + """Dirty dirty method to normalize the error messages between the API calls. + Any response containing the a key 'error' or 'errors' failed at some point, + we make one single list out of it. + """ messages = [] if response.get('error'): if isinstance(response['error'], list): @@ -204,6 +208,7 @@ class PyMISP(object): return messages def _check_response(self, response): + """Check if the response from the server is not an unexpected error""" if response.status_code >= 500: response.raise_for_status() try: @@ -243,6 +248,7 @@ class PyMISP(object): return value if isinstance(value, (tuple, list)) else (value,) def _make_mispevent(self, event): + """Transform a Json MISP event into a MISPEvent""" if not isinstance(event, MISPEvent): e = MISPEvent(self.describe_types) e.load(event) @@ -251,6 +257,7 @@ class PyMISP(object): return e def _prepare_full_event(self, distribution, threat_level_id, analysis, info, date=None, published=False, orgc_id=None, org_id=None, sharing_group_id=None): + """Initialize a new MISPEvent from scratch""" misp_event = MISPEvent(self.describe_types) misp_event.set_all_values(info=info, distribution=distribution, threat_level_id=threat_level_id, analysis=analysis, date=date, orgc_id=orgc_id, org_id=org_id, sharing_group_id=sharing_group_id) @@ -259,6 +266,7 @@ class PyMISP(object): return misp_event def _prepare_full_attribute(self, category, type_value, value, to_ids, comment=None, distribution=5, **kwargs): + """Initialize a new MISPAttribute from scratch""" misp_attribute = MISPAttribute(self.describe_types) misp_attribute.set_all_values(type=type_value, value=value, category=category, to_ids=to_ids, comment=comment, distribution=distribution, **kwargs) @@ -353,12 +361,14 @@ class PyMISP(object): return self._check_response(response) def delete_attribute(self, attribute_id): + """Delete an attribute by ID""" session = self.__prepare_session() url = urljoin(self.root_url, 'attributes/{}'.format(attribute_id)) response = session.delete(url) return self._check_response(response) def pushEventToZMQ(self, event_id): + """Force push an event on ZMQ""" session = self.__prepare_session() url = urljoin(self.root_url, 'events/pushEventToZMQ/{}.json'.format(event_id)) response = session.post(url) @@ -369,9 +379,11 @@ class PyMISP(object): # ############################################## def get(self, eid): + """Get an event by event ID""" return self.get_event(eid) def update(self, event): + """Update an event by ID""" e = self._make_mispevent(event) if e.uuid: eid = e.uuid @@ -401,21 +413,25 @@ class PyMISP(object): return self._check_response(response) def change_threat_level(self, event, threat_level_id): + """Change the threat level of an event""" e = self._make_mispevent(event) e.threat_level_id = threat_level_id return self.update(e) def change_sharing_group(self, event, sharing_group_id): + """Change the sharing group of an event""" e = self._make_mispevent(event) e.distribution = 4 # Needs to be 'Sharing group' e.sharing_group_id = sharing_group_id return self.update(e) def new_event(self, distribution=None, threat_level_id=None, analysis=None, info=None, date=None, published=False, orgc_id=None, org_id=None, sharing_group_id=None): + """Create and add a new event""" misp_event = self._prepare_full_event(distribution, threat_level_id, analysis, info, date, published, orgc_id, org_id, sharing_group_id) return self.add_event(misp_event) def tag(self, uuid, tag): + """Tag an event or an attribute""" if not self._valid_uuid(uuid): raise PyMISPError('Invalid UUID') session = self.__prepare_session() @@ -425,6 +441,7 @@ class PyMISP(object): return self._check_response(response) def untag(self, uuid, tag): + """Untag an event or an attribute""" if not self._valid_uuid(uuid): raise PyMISPError('Invalid UUID') session = self.__prepare_session() @@ -436,6 +453,7 @@ class PyMISP(object): # ##### File attributes ##### def _send_attributes(self, event, attributes, proposal=False): + """Helper to add new attributes to an existing events""" eventID_to_update = None if isinstance(event, MISPEvent): if hasattr(event, 'id'): @@ -465,12 +483,14 @@ class PyMISP(object): return response def add_named_attribute(self, event, type_value, value, category=None, to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): + """Add one or more attributes to an existing event""" attributes = [] for value in self._one_or_more(value): attributes.append(self._prepare_full_attribute(category, type_value, value, to_ids, comment, distribution, **kwargs)) return self._send_attributes(event, attributes, proposal) def add_hashes(self, event, category='Artifacts dropped', filename=None, md5=None, sha1=None, sha256=None, ssdeep=None, comment=None, to_ids=True, distribution=None, proposal=False, **kwargs): + """Add hashe(s) to an existing event""" attributes = [] type_value = '{}' @@ -490,12 +510,15 @@ class PyMISP(object): return self._send_attributes(event, attributes, proposal) def av_detection_link(self, event, link, category='Antivirus detection', to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): + """Add AV detection link(s)""" return self.add_named_attribute(event, 'link', link, category, to_ids, comment, distribution, proposal, **kwargs) def add_detection_name(self, event, name, category='Antivirus detection', to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): + """Add AV detection name(s)""" return self.add_named_attribute(event, 'text', name, category, to_ids, comment, distribution, proposal, **kwargs) def add_filename(self, event, filename, category='Artifacts dropped', to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): + """Add filename(s)""" return self.add_named_attribute(event, 'filename', filename, category, to_ids, comment, distribution, proposal, **kwargs) def add_attachment(self, event, attachment, category='Artifacts dropped', to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): @@ -538,6 +561,7 @@ class PyMISP(object): return self.add_named_attribute(event, 'attachment', filename, category, to_ids, comment, distribution, proposal, data=encodedData, **kwargs) def add_regkey(self, event, regkey, rvalue=None, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): + """Add a registry key""" if rvalue: type_value = 'regkey|value' value = '{}|{}'.format(regkey, rvalue) @@ -550,6 +574,7 @@ class PyMISP(object): return self._send_attributes(event, attributes, proposal) def add_regkeys(self, event, regkeys_values, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): + """Add a registry keys""" attributes = [] for regkey, rvalue in regkeys_values.items(): @@ -564,12 +589,14 @@ class PyMISP(object): return self._send_attributes(event, attributes, proposal) def add_pattern(self, event, pattern, in_file=True, in_memory=False, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): + """Add a pattern(s) in file or in memory""" if not (in_file or in_memory): raise PyMISPError('Invalid pattern type: please use in_memory=True or in_file=True') itemtype = 'pattern-in-file' if in_file else 'pattern-in-memory' return self.add_named_attribute(event, itemtype, pattern, category, to_ids, comment, distribution, proposal, **kwargs) def add_pipe(self, event, named_pipe, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): + """Add pipes(s)""" def scrub(s): if not s.startswith('\\.\\pipe\\'): s = '\\.\\pipe\\{}'.format(s) @@ -578,6 +605,7 @@ class PyMISP(object): return self.add_named_attribute(event, 'named pipe', attributes, category, to_ids, comment, distribution, proposal, **kwargs) def add_mutex(self, event, mutex, category='Artifacts dropped', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): + """Add mutex(es)""" def scrub(s): if not s.startswith('\\BaseNamedObjects\\'): s = '\\BaseNamedObjects\\{}'.format(s) @@ -586,98 +614,125 @@ class PyMISP(object): return self.add_named_attribute(event, 'mutex', attributes, category, to_ids, comment, distribution, proposal, **kwargs) def add_yara(self, event, yara, category='Payload delivery', to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): + """Add yara rule(es)""" return self.add_named_attribute(event, 'yara', yara, category, to_ids, comment, distribution, proposal, **kwargs) # ##### Network attributes ##### def add_ipdst(self, event, ipdst, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): + """Add destination IP(s)""" return self.add_named_attribute(event, 'ip-dst', ipdst, category, to_ids, comment, distribution, proposal, **kwargs) def add_ipsrc(self, event, ipsrc, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): + """Add source IP(s)""" return self.add_named_attribute(event, 'ip-src', ipsrc, category, to_ids, comment, distribution, proposal, **kwargs) def add_hostname(self, event, hostname, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): + """Add hostname(s)""" return self.add_named_attribute(event, 'hostname', hostname, category, to_ids, comment, distribution, proposal, **kwargs) def add_domain(self, event, domain, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): + """Add domain(s)""" return self.add_named_attribute(event, 'domain', domain, category, to_ids, comment, distribution, proposal, **kwargs) def add_domain_ip(self, event, domain, ip, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): + """Add domain|ip""" if isinstance(ip, str): ip = [ip] composed = list(map(lambda x: '%s|%s' % (domain, x), ip)) return self.add_named_attribute(event, 'domain|ip', composed, category, to_ids, comment, distribution, proposal, **kwargs) def add_domains_ips(self, event, domain_ips, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): + """Add multiple domain|ip""" composed = list(map(lambda x: '%s|%s' % (x[0], x[1]), domain_ips.items())) return self.add_named_attribute(event, 'domain|ip', composed, category, to_ids, comment, distribution, proposal, **kwargs) def add_url(self, event, url, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): + """Add url(s)""" return self.add_named_attribute(event, 'url', url, category, to_ids, comment, distribution, proposal, **kwargs) def add_useragent(self, event, useragent, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): + """Add user agent(s)""" return self.add_named_attribute(event, 'user-agent', useragent, category, to_ids, comment, distribution, proposal, **kwargs) def add_traffic_pattern(self, event, pattern, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): + """Add pattern(s) in traffic""" return self.add_named_attribute(event, 'pattern-in-traffic', pattern, category, to_ids, comment, distribution, proposal, **kwargs) def add_snort(self, event, snort, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): + """Add SNORT rule(s)""" return self.add_named_attribute(event, 'snort', snort, category, to_ids, comment, distribution, proposal, **kwargs) def add_net_other(self, event, netother, category='Network activity', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): + """Add a free text entry""" return self.add_named_attribute(event, 'other', netother, category, to_ids, comment, distribution, proposal, **kwargs) # ##### Email attributes ##### def add_email_src(self, event, email, category='Payload delivery', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): + """Add a source email""" return self.add_named_attribute(event, 'email-src', email, category, to_ids, comment, distribution, proposal, **kwargs) def add_email_dst(self, event, email, category='Payload delivery', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): + """Add a destination email""" return self.add_named_attribute(event, 'email-dst', email, category, to_ids, comment, distribution, proposal, **kwargs) def add_email_subject(self, event, email, category='Payload delivery', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): + """Add an email subject""" return self.add_named_attribute(event, 'email-subject', email, category, to_ids, comment, distribution, proposal, **kwargs) def add_email_attachment(self, event, email, category='Payload delivery', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): + """Add an email atachment""" return self.add_named_attribute(event, 'email-attachment', email, category, to_ids, comment, distribution, proposal, **kwargs) # ##### Target attributes ##### def add_target_email(self, event, target, category='Targeting data', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): + """Add an target email""" return self.add_named_attribute(event, 'target-email', target, category, to_ids, comment, distribution, proposal, **kwargs) def add_target_user(self, event, target, category='Targeting data', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): + """Add an target user""" return self.add_named_attribute(event, 'target-user', target, category, to_ids, comment, distribution, proposal, **kwargs) def add_target_machine(self, event, target, category='Targeting data', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): + """Add an target machine""" return self.add_named_attribute(event, 'target-machine', target, category, to_ids, comment, distribution, proposal, **kwargs) def add_target_org(self, event, target, category='Targeting data', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): + """Add an target organisation""" return self.add_named_attribute(event, 'target-org', target, category, to_ids, comment, distribution, proposal, **kwargs) def add_target_location(self, event, target, category='Targeting data', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): + """Add an target location""" return self.add_named_attribute(event, 'target-location', target, category, to_ids, comment, distribution, proposal, **kwargs) def add_target_external(self, event, target, category='Targeting data', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): + """Add an target external""" return self.add_named_attribute(event, 'target-external', target, category, to_ids, comment, distribution, proposal, **kwargs) # ##### Attribution attributes ##### def add_threat_actor(self, event, target, category='Attribution', to_ids=True, comment=None, distribution=None, proposal=False, **kwargs): + """Add an threat actor""" return self.add_named_attribute(event, 'threat-actor', target, category, to_ids, comment, distribution, proposal, **kwargs) # ##### Internal reference attributes ##### def add_internal_link(self, event, reference, category='Internal reference', to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): + """Add an internal link""" return self.add_named_attribute(event, 'link', reference, category, to_ids, comment, distribution, proposal, **kwargs) def add_internal_comment(self, event, reference, category='Internal reference', to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): + """Add an internal comment""" return self.add_named_attribute(event, 'comment', reference, category, to_ids, comment, distribution, proposal, **kwargs) def add_internal_text(self, event, reference, category='Internal reference', to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): + """Add an internal text""" return self.add_named_attribute(event, 'text', reference, category, to_ids, comment, distribution, proposal, **kwargs) def add_internal_other(self, event, reference, category='Internal reference', to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): + """Add an internal reference (type other)""" return self.add_named_attribute(event, 'other', reference, category, to_ids, comment, distribution, proposal, **kwargs) # ################################################## @@ -686,6 +741,7 @@ class PyMISP(object): def _prepare_upload(self, event_id, distribution, to_ids, category, comment, info, analysis, threat_level_id): + """Helper to prepare a sample to upload""" to_post = {'request': {}} if event_id is not None: @@ -716,6 +772,7 @@ class PyMISP(object): return to_post def _encode_file_to_upload(self, filepath_or_bytes): + """Helper to encode a file to upload""" if isinstance(filepath_or_bytes, basestring) and os.path.isfile(filepath_or_bytes): with open(filepath_or_bytes, 'rb') as f: binblob = f.read() @@ -726,6 +783,7 @@ class PyMISP(object): def upload_sample(self, filename, filepath_or_bytes, event_id, distribution=None, to_ids=True, category=None, comment=None, info=None, analysis=None, threat_level_id=None): + """Upload a sample""" to_post = self._prepare_upload(event_id, distribution, to_ids, category, comment, info, analysis, threat_level_id) to_post['request']['files'] = [{'filename': filename, 'data': self._encode_file_to_upload(filepath_or_bytes)}] @@ -734,6 +792,7 @@ class PyMISP(object): def upload_samplelist(self, filepaths, event_id, distribution=None, to_ids=True, category=None, comment=None, info=None, analysis=None, threat_level_id=None): + """Upload a list of samples""" to_post = self._prepare_upload(event_id, distribution, to_ids, category, comment, info, analysis, threat_level_id) files = [] @@ -745,6 +804,7 @@ class PyMISP(object): return self._upload_sample(to_post) def _upload_sample(self, to_post): + """Helper to upload a sample""" session = self.__prepare_session() url = urljoin(self.root_url, 'events/upload_sample') response = session.post(url, data=json.dumps(to_post)) @@ -755,6 +815,7 @@ class PyMISP(object): # ############################ def __query_proposal(self, session, path, id, attribute=None): + """Helper to prepare a query to handle proposals""" url = urljoin(self.root_url, 'shadow_attributes/{}/{}'.format(path, id)) if path in ['add', 'edit']: query = {'request': {'ShadowAttribute': attribute}} @@ -766,6 +827,7 @@ class PyMISP(object): return self._check_response(response) def proposal_view(self, event_id=None, proposal_id=None): + """View a proposal""" session = self.__prepare_session() if proposal_id is not None and event_id is not None: return {'error': 'You can only view an event ID or a proposal ID'} @@ -776,18 +838,22 @@ class PyMISP(object): return self.__query_proposal(session, 'view', id) def proposal_add(self, event_id, attribute): + """Add a proposal""" session = self.__prepare_session() return self.__query_proposal(session, 'add', event_id, attribute) def proposal_edit(self, attribute_id, attribute): + """Edit a proposal""" session = self.__prepare_session() return self.__query_proposal(session, 'edit', attribute_id, attribute) def proposal_accept(self, proposal_id): + """Accept a proposal""" session = self.__prepare_session() return self.__query_proposal(session, 'accept', proposal_id) def proposal_discard(self, proposal_id): + """Discard a proposal""" session = self.__prepare_session() return self.__query_proposal(session, 'discard', proposal_id) @@ -796,6 +862,7 @@ class PyMISP(object): # ############################## def change_toids(self, attribute_uuid, to_ids): + """Change the toids flag""" if to_ids not in [0, 1]: raise Exception('to_ids can only be 0 or 1') query = {"to_ids": to_ids} @@ -807,6 +874,7 @@ class PyMISP(object): # ############################## def freetext(self, event_id, string, adhereToWarninglists=False, distribution=None): + """Pass a text to the freetext importer""" query = {"value": string} wl_params = [False, True, 'soft'] if adhereToWarninglists not in wl_params: @@ -823,6 +891,7 @@ class PyMISP(object): # ############################## def __query(self, session, path, query, controller='events', async_callback=None): + """Helper to prepare a search query""" if query.get('error') is not None: return query if controller not in ['events', 'attributes']: @@ -897,6 +966,7 @@ class PyMISP(object): return res def search_all(self, value): + """Search a value in the whole database""" query = {'value': value, 'searchall': 1} session = self.__prepare_session() return self.__query(session, 'restSearch/download', query) @@ -1030,6 +1100,7 @@ class PyMISP(object): return response.content def get_yara(self, event_id): + """Get the yara rules from an event""" to_post = {'request': {'eventid': event_id, 'type': 'yara'}} session = self.__prepare_session() response = session.post(urljoin(self.root_url, 'attributes/restSearch'), data=json.dumps(to_post)) @@ -1042,6 +1113,7 @@ class PyMISP(object): return True, rules def download_samples(self, sample_hash=None, event_id=None, all_samples=False): + """Download samples, by hash or event ID. If there are multiple samples in one event, use the all_samples switch""" to_post = {'request': {'hash': sample_hash, 'eventID': event_id, 'allSamples': all_samples}} session = self.__prepare_session() response = session.post(urljoin(self.root_url, 'attributes/downloadSample'), data=json.dumps(to_post)) @@ -1079,6 +1151,7 @@ class PyMISP(object): # ########## Tags ########## def get_all_tags(self, quiet=False): + """Get all the tags used on the instance""" session = self.__prepare_session() url = urljoin(self.root_url, 'tags') response = session.get(url) @@ -1092,6 +1165,7 @@ class PyMISP(object): return to_return def new_tag(self, name=None, colour="#00ace6", exportable=False): + """Create a new tag""" to_post = {'Tag': {'name': name, 'colour': colour, 'exportable': exportable}} session = self.__prepare_session() url = urljoin(self.root_url, 'tags/add') @@ -1168,18 +1242,21 @@ class PyMISP(object): # ############## Sightings ################## def sighting_per_id(self, attribute_id): + """Add a sighting to an attribute (by attribute ID)""" session = self.__prepare_session() url = urljoin(self.root_url, 'sightings/add/{}'.format(attribute_id)) response = session.post(url) return self._check_response(response) def sighting_per_uuid(self, attribute_uuid): + """Add a sighting to an attribute (by attribute UUID)""" session = self.__prepare_session() url = urljoin(self.root_url, 'sightings/add/{}'.format(attribute_uuid)) response = session.post(url) return self._check_response(response) def set_sightings(self, sightings): + """Push a sighting (python dictionary)""" if isinstance(sightings, dict): sightings = json.dumps(sightings) session = self.__prepare_session() @@ -1188,6 +1265,7 @@ class PyMISP(object): return self._check_response(response) def sighting_per_json(self, json_file): + """Push a sighting (JSON file)""" with open(json_file, 'r') as f: jdata = json.load(f) return self.set_sightings(jdata) @@ -1195,6 +1273,7 @@ class PyMISP(object): # ############## Sharing Groups ################## def get_sharing_groups(self): + """Get the existing sharing groups""" session = self.__prepare_session() url = urljoin(self.root_url, 'sharing_groups.json') response = session.get(url) @@ -1474,6 +1553,7 @@ class PyMISP(object): # ############## Roles ################## def get_roles_list(self): + """Get the list of existing roles""" session = self.__prepare_session() url = urljoin(self.root_url, '/roles') response = session.get(url) @@ -1482,6 +1562,7 @@ class PyMISP(object): # ############## Tags ################## def get_tags_list(self): + """Get the list of existing tags""" session = self.__prepare_session() url = urljoin(self.root_url, '/tags') response = session.get(url) @@ -1542,86 +1623,93 @@ class PyMISP(object): # ########################### def fetch_feed(self, feed_id): + """Fetch one single feed""" session = self.__prepare_session() url = urljoin(self.root_url, 'feeds/fetchFromFeed/{}'.format(feed_id)) response = session.get(url) return self._check_response(response) def view_feeds(self): + """Get the content of all the feeds""" session = self.__prepare_session() url = urljoin(self.root_url, 'feeds') response = session.get(url) return self._check_response(response) def view_feed(self, feed_ids): + """Get the content of a single feed""" session = self.__prepare_session() url = urljoin(self.root_url, 'feeds/view/{}'.format(feed_ids)) response = session.get(url) return self._check_response(response) def cache_feeds_all(self): + """ Cache all the feeds""" session = self.__prepare_session() url = urljoin(self.root_url, 'feeds/cacheFeeds/all') response = session.get(url) return self._check_response(response) def cache_feed(self, feed_id): + """Cache a specific feed""" session = self.__prepare_session() url = urljoin(self.root_url, 'feeds/cacheFeeds/{}'.format(feed_id)) response = session.get(url) return self._check_response(response) def cache_feeds_freetext(self): + """Cache all the freetext feeds""" session = self.__prepare_session() url = urljoin(self.root_url, 'feeds/cacheFeeds/freetext') response = session.get(url) return self._check_response(response) def cache_feeds_misp(self): + """Cache all the MISP feeds""" session = self.__prepare_session() url = urljoin(self.root_url, 'feeds/cacheFeeds/misp') response = session.get(url) return self._check_response(response) def compare_feeds(self): + """Generate the comparison matrix for all the MISP feeds""" session = self.__prepare_session() url = urljoin(self.root_url, 'feeds/compareFeeds') response = session.get(url) return self._check_response(response) - # ########################### - # ### Cache All Feeds ### - # ########################### - def cache_all_feeds(self): - session = self.__prepare_session() - url = urljoin(self.root_url, 'feeds/cacheFeeds/all') - response = session.post(url) - return self._check_response(response) + """Alias for cache_feeds_all""" + # DEPRECATED + return self.cache_feeds_all() # ################### # ### Objects ### # ################### def add_object(self, event_id, template_id, misp_object): + """Add an object""" session = self.__prepare_session() url = urljoin(self.root_url, 'objects/add/{}/{}'.format(event_id, template_id)) response = session.post(url, data=misp_object.to_json()) return self._check_response(response) def add_object_reference(self, misp_object_reference): + """Add a reference to an object""" session = self.__prepare_session() url = urljoin(self.root_url, 'object_references/add') response = session.post(url, data=misp_object_reference.to_json()) return self._check_response(response) def get_object_templates_list(self): + """Returns the list of Object templates available on the MISP instance""" session = self.__prepare_session() url = urljoin(self.root_url, 'objectTemplates') response = session.get(url) return self._check_response(response)['response'] def get_object_template_id(self, object_uuid): + """Gets the template ID corresponting the UUID passed as parameter""" templates = self.get_object_templates_list() for t in templates: if t['ObjectTemplate']['uuid'] == object_uuid: From a63d53094f58f90406530b0cbb9c6a8ec10769fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sun, 22 Oct 2017 12:17:48 -0400 Subject: [PATCH 09/36] fix: Fix typos and logic mistakes in mispevent. --- pymisp/mispevent.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index cc4c74a..0e99ed2 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -165,10 +165,15 @@ class MISPAttribute(AbstractMISP): # Default values self.category = kwargs.pop('category', type_defaults['default_category']) + if self.category is None: + # In case the category key is passed, but None + self.category = type_defaults['default_category'] if self.category not in self.__categories: raise NewAttributeError('{} is invalid, category has to be in {}'.format(self.category, (', '.join(self.__categories)))) self.to_ids = kwargs.pop('to_ids', bool(int(type_defaults['to_ids']))) + if self.to_ids is None: + self.to_ids = bool(int(type_defaults['to_ids'])) if not isinstance(self.to_ids, bool): raise NewAttributeError('{} is invalid, to_ids has to be True or False'.format(self.to_ids)) @@ -565,7 +570,7 @@ class MISPObjectReference(AbstractMISP): self.referenced_uuid = referenced_uuid self.relationship_type = relationship_type self.comment = comment - for k, v in kwargs: + for k, v in kwargs.items(): setattr(self, k, v) @@ -684,7 +689,7 @@ class MISPObject(AbstractMISP): """Add a link (uuid) to an other object""" if kwargs.get('object_uuid'): # Load existing object - object_uuid = kwargs.get('object_uuid') + object_uuid = kwargs.pop('object_uuid') else: # New reference object_uuid = self.uuid From 4c4cd23983539a26482497aa81c81219b80ddc77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sun, 22 Oct 2017 14:02:47 -0400 Subject: [PATCH 10/36] chg: Allow to hard delete an attribute by ID. --- pymisp/api.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 83105c6..adc9784 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -360,11 +360,14 @@ class PyMISP(object): response = session.delete(url) return self._check_response(response) - def delete_attribute(self, attribute_id): + def delete_attribute(self, attribute_id, hard_delete=False): """Delete an attribute by ID""" session = self.__prepare_session() - url = urljoin(self.root_url, 'attributes/{}'.format(attribute_id)) - response = session.delete(url) + if hard_delete: + url = urljoin(self.root_url, 'attributes/delete/{}/1'.format(attribute_id)) + else: + url = urljoin(self.root_url, 'attributes/delete/{}'.format(attribute_id)) + response = session.get(url) return self._check_response(response) def pushEventToZMQ(self, event_id): From cb166a6295297f462ac3b8bb1277eec9fd128973 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 23 Oct 2017 11:53:21 -0400 Subject: [PATCH 11/36] chg: Bump misp-objects --- pymisp/data/misp-objects | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index a5d2f71..b801bc6 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit a5d2f71fef1552b6957fc74b4c2a8650355becae +Subproject commit b801bc6603c9ba9f6bec5b5868ae26b89a11d862 From 8f16f741cf82d350c1cb54f6231180799141f0e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 23 Oct 2017 14:08:32 -0400 Subject: [PATCH 12/36] fix: Properly bundle object templates --- MANIFEST.in | 5 ++++- setup.py | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index d1cf49c..4da3e82 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1,4 @@ -include pymisp/data/* +include pymisp/data/*.json +include pymisp/data/misp-objects/*.json +include pymisp/misp-objects/objects/*/definition.json +include pymisp/misp-objects/relationships/definition.json diff --git a/setup.py b/setup.py index 64b5e1a..2d5245f 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,8 @@ setup( 'six' ], include_package_data=True, - package_data={'pymisp': ['data/*.json', 'data/misp-objects/schema_objects.json', + package_data={'pymisp': ['data/*.json', 'data/misp-objects', + 'data/misp-objects/schema_objects.json', 'data/misp-objects/schema_relationships.json', 'data/misp-objects/objects/*/definition.json', 'data/misp-objects/relationships/definition.json']}, From 85669abbcbd18178c8b7745ffc649bdf4eed3481 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 24 Oct 2017 08:52:00 -0400 Subject: [PATCH 13/36] chg: Do not raise an exception when the object template is unknown. + bump misp-object --- pymisp/data/misp-objects | 2 +- pymisp/mispevent.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index b801bc6..bbf3e45 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit b801bc6603c9ba9f6bec5b5868ae26b89a11d862 +Subproject commit bbf3e45649af5af50c98ad90a86916cf75e8c74d diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 0e99ed2..f00c5cf 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -603,7 +603,7 @@ class MISPObjectAttribute(MISPAttribute): class MISPObject(AbstractMISP): - def __init__(self, name, strict=True): + def __init__(self, name, strict=False): super(MISPObject, self).__init__() self.__strict = strict self.name = name @@ -639,7 +639,7 @@ class MISPObject(AbstractMISP): else: self.__known_template = False if kwargs.get('template_version') and int(kwargs['template_version']) != self.template_version: - if self.strict: + if self.__strict: raise UnknownMISPObjectTemplate('Version of the object ({}) is different from the one of the template ({}).'.format(kwargs['template_version'], self.template_version)) else: self.__known_template = False @@ -654,12 +654,12 @@ class MISPObject(AbstractMISP): else: setattr(self, key, value) - def to_dict(self, strict=True): + def to_dict(self, strict=False): if strict or self.__strict and self.__known_template: self._validate() return super(MISPObject, self).to_dict() - def to_json(self, strict=True): + def to_json(self, strict=False): if strict or self.__strict and self.__known_template: self._validate() return super(MISPObject, self).to_json() From 7686020f95bbd7e6b90f45063891e488b7e1045b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 24 Oct 2017 08:54:29 -0400 Subject: [PATCH 14/36] chg: Bump version --- pymisp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 88fa845..a318a2d 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.4.81' +__version__ = '2.4.81.1' try: from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate # noqa From 066a9836e0653f09890b62fa03c7ab06c4498b69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 24 Oct 2017 08:56:08 -0400 Subject: [PATCH 15/36] chg: update changelog --- CHANGELOG.txt | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 562eeb0..0bf78e8 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -2,6 +2,48 @@ Changelog ========= +v2.4.81.1 (2017-10-24) +---------------------- + +Changes +~~~~~~~ +- Bump version. [Raphaël Vinot] +- Do not raise an exception when the object template is unknown. + [Raphaël Vinot] + + + bump misp-object +- Bump misp-objects. [Raphaël Vinot] +- Allow to hard delete an attribute by ID. [Raphaël Vinot] +- Update comments. [Raphaël Vinot] +- Bump misp-objects and describeTypes. [Raphaël Vinot] + +Fix +~~~ +- Properly bundle object templates. [Raphaël Vinot] +- Fix typos and logic mistakes in mispevent. [Raphaël Vinot] +- Fix travis build. [Raphaël Vinot] +- Min required version of setuptools. [Raphaël Vinot] +- Improve dependencies listing. [Raphaël Vinot] + + Partial fix for #110 +- Missing default category. [Raphaël Vinot] + + Fix #119 + +Other +~~~~~ +- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot] +- Update openioc.py. [Andras Iklody] +- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot] +- Merge pull request #121 from kx499/master. [Raphaël Vinot] + + Added **kwargs to add_named_attribute call in add_attachment +- Added **kwargs to add_named_attribute call in add_attachment. + [tssbo82] +- Update README. [Raphaël Vinot] +- Update changelog. [Raphaël Vinot] + + v2.4.81 (2017-10-09) -------------------- From 39ffa91cd441a3b45e5deb95e7557498800db0f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 24 Oct 2017 09:07:33 -0400 Subject: [PATCH 16/36] fix: properly bundle object templates --- MANIFEST.in | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 4da3e82..dd52be9 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ include pymisp/data/*.json include pymisp/data/misp-objects/*.json -include pymisp/misp-objects/objects/*/definition.json -include pymisp/misp-objects/relationships/definition.json +include pymisp/data/misp-objects/objects/*/definition.json +include pymisp/data/misp-objects/relationships/definition.json diff --git a/setup.py b/setup.py index 2d5245f..528b2d6 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ setup( 'six' ], include_package_data=True, - package_data={'pymisp': ['data/*.json', 'data/misp-objects', + package_data={'pymisp': ['data/*.json', 'data/misp-objects/schema_objects.json', 'data/misp-objects/schema_relationships.json', 'data/misp-objects/objects/*/definition.json', From 23f82e68cd5f90ae73124c3516ff0d2c9ef12d96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 24 Oct 2017 09:07:57 -0400 Subject: [PATCH 17/36] chg: version bump --- pymisp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index a318a2d..55d2a19 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.4.81.1' +__version__ = '2.4.81.2' try: from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate # noqa From cb94cda923bc0e399838cfbf0720d461d1f56943 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 24 Oct 2017 09:08:51 -0400 Subject: [PATCH 18/36] chg: Update changelog --- CHANGELOG.txt | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 0bf78e8..f6e5ebf 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -2,6 +2,19 @@ Changelog ========= +v2.4.81.2 (2017-10-24) +---------------------- + +Changes +~~~~~~~ +- Version bump. [Raphaël Vinot] +- Update changelog. [Raphaël Vinot] + +Fix +~~~ +- Properly bundle object templates. [Raphaël Vinot] + + v2.4.81.1 (2017-10-24) ---------------------- From 6517081fabbf2e313c1f82d3628a2303711132b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 24 Oct 2017 18:09:10 -0400 Subject: [PATCH 19/36] chg: Add simple asciidoc generator for MISP event --- examples/asciidoc_generator.py | 139 +++++++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100755 examples/asciidoc_generator.py diff --git a/examples/asciidoc_generator.py b/examples/asciidoc_generator.py new file mode 100755 index 0000000..156a7ff --- /dev/null +++ b/examples/asciidoc_generator.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from pymisp import MISPEvent +from defang import defang +import argparse +from pytaxonomies import Taxonomies +from datetime import date + +headers = """ +:toc: right +:toclevels: 1 +:toc-title: Daily Report +:icons: font +:sectanchors: +:sectlinks: += Daily report by {org_name} +{date} + +:icons: font + +""" + +event_level_tags = """ +IMPORTANT: This event is classified TLP:{value}. + +{expanded} + +""" + +attributes = """ +=== Indicator(s) of compromise + +{list_attributes} + +""" + +title = """ +== ({internal_id}) {title} + +{summary} + +""" + +types_to_attach = ['ip-dst', 'url', 'domain'] +objects_to_attach = ['domain-ip'] + +class ReportGenerator(): + + def __init__(self): + self.taxonomies = Taxonomies() + self.report = '' + + def from_remote(self, event_id): + from pymisp import PyMISP + from keys import misp_url, misp_key, misp_verifycert + misp = PyMISP(misp_url, misp_key, misp_verifycert) + result = misp.get(event_id) + self.misp_event = MISPEvent() + self.misp_event.load(result) + + def from_file(self, path): + self.misp_event = MISPEvent() + self.misp_event.load_file(path) + + def attributes(self): + if not self.misp_event.attributes: + return '' + list_attributes = '' + for attribute in self.misp_event.attributes: + if attribute.type in types_to_attach: + list_attributes += "\n* {}\n".format(defang(attribute.value)) + for obj in self.misp_event.Object: + for attribute in obj.Attribute: + if attribute.type in types_to_attach: + list_attributes += "\n* {}\n".format(defang(attribute.value)) + return attributes.format(list_attributes=list_attributes) + + def _get_tag_info(self, machinetag): + return self.taxonomies.revert_machinetag(machinetag) + + def report_headers(self): + content = {'org_name': 'name', + 'date': date.today().isoformat()} + self.report += headers.format(**content) + + def event_level_tags(self): + if not self.misp_event.Tag: + return '' + for tag in self.misp_event.Tag: + # Only look for TLP for now + if tag['name'].startswith('tlp'): + tax, predicate = self._get_tag_info(tag['name']) + return event_level_tags.format(value=predicate.predicate.upper(), expanded=predicate.expanded) + + def title(self): + internal_id = '' + summary = '' + # Get internal refs for report + for obj in self.misp_event.Object: + if obj.name != 'report': + continue + for a in obj.Attribute: + if a.object_relation == 'case-number': + internal_id = a.value + if a.object_relation == 'summary': + summary = a.value + + return title.format(internal_id=internal_id, title=self.misp_event.info, + summary=summary) + + + def asciidoc(self, lang='en'): + self.report += self.title() + self.report += self.event_level_tags() + self.report += self.attributes() + +if __name__ == '__main__': + + parser = argparse.ArgumentParser(description='Create a human-readable report out of a MISP event') + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("-e", "--event", default=[], nargs='+', help="Event ID to get.") + group.add_argument("-p", "--path", default=[], nargs='+', help="Path to the JSON dump.") + + args = parser.parse_args() + + report = ReportGenerator() + report.report_headers() + + if args.event: + for eid in args.event: + report.from_remote(eid) + report.asciidoc() + else: + for f in args.path: + report.from_file(f) + report.asciidoc() + + print(report.report) From 4152435250b9eddc0072410337ebc847ce1e46b4 Mon Sep 17 00:00:00 2001 From: garanews Date: Fri, 20 Oct 2017 09:55:46 +0200 Subject: [PATCH 20/36] Created add_generic_object.py usage: add_generic_object.py [-h] -e EVENT -t TYPE -d DICT Examples: python3 add_generic_object.py -e 1683 -t email -d '{"subject":"The Pink Letter", "to":"jon@snow.org"}' python3 add_generic_object.py -e 2343 -t person -d '{"first-name":"Daenerys", "last-name":"Targaryen", "place-of-birth":"Dragonstone"}' python3 add_generic_object.py -e 3596 -t "domain|ip" -d '{"domain":"stormborn.org", "ip":"50.63.202.33"}' --- examples/add_generic_object.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 examples/add_generic_object.py diff --git a/examples/add_generic_object.py b/examples/add_generic_object.py new file mode 100644 index 0000000..308a1a3 --- /dev/null +++ b/examples/add_generic_object.py @@ -0,0 +1,33 @@ +import json +from pymisp import PyMISP +from pymisp.tools.abstractgenerator import AbstractMISPObjectGenerator +from keys import misp_url, misp_key, misp_verifycert +import argparse + +class GenericObject(AbstractMISPObjectGenerator): + def __init__(self, type, data_dict): + super(GenericObject, self).__init__(type) + self.__data = data_dict + self.generate_attributes() + + def generate_attributes(self): + for key, value in self.__data.items(): + self.add_attribute(key, value=value) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Create a MISP Object selectable by type starting from a dictionary') + parser.add_argument("-e", "--event", required=True, help="Event ID to update") + parser.add_argument("-t", "--type", required=True, help="Type of the generic object") + parser.add_argument("-d", "--dict", required=True, help="Dict ") + args = parser.parse_args() + + pymisp = PyMISP(misp_url, misp_key, misp_verifycert) + try: + template_id = [x['ObjectTemplate']['id'] for x in pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == args.type][0] + except IndexError: + valid_types = ", ".join([x['ObjectTemplate']['name'] for x in pymisp.get_object_templates_list()]) + print ("Template for type %s not found! Valid types are: %s" % (args.type, valid_types)) + exit() + + misp_object = GenericObject(args.type.replace("|", "-"), json.loads(args.dict)) + r = pymisp.add_object(args.event, template_id, misp_object) From e2d690d0ef59da18267af3755c43236d09a20f09 Mon Sep 17 00:00:00 2001 From: Thomas Gardner Date: Wed, 25 Oct 2017 09:48:18 -0400 Subject: [PATCH 21/36] added vtreportobject and vt_to_misp example --- examples/vt_to_misp.py | 182 +++++++++++++++++++++++++++++++++ pymisp/tools/__init__.py | 1 + pymisp/tools/vtreportobject.py | 74 ++++++++++++++ 3 files changed, 257 insertions(+) create mode 100644 examples/vt_to_misp.py create mode 100644 pymisp/tools/vtreportobject.py diff --git a/examples/vt_to_misp.py b/examples/vt_to_misp.py new file mode 100644 index 0000000..5554aba --- /dev/null +++ b/examples/vt_to_misp.py @@ -0,0 +1,182 @@ +''' Convert a VirusTotal report into MISP objects ''' +import argparse +import json +import logging +from datetime import datetime +from urllib.parse import urlsplit + +import pymisp +from pymisp.tools import VTReportObject + +logging.basicConfig(level=logging.INFO, format="%(asctime)s | %(levelname)s | %(module)s.%(funcName)s.%(lineno)d | %(message)s") + + +def build_cli(): + ''' + Build the command-line arguments + ''' + desc = "Take an indicator or list of indicators to search VT for and import the results into MISP" + post_desc = """ +config.json: Should be a JSON file containing MISP and VirusTotal credentials with the following format: +{"misp": {"url": "", "key": ""}, "virustotal": {"key": ""}} +Please note: Only public API features work in the VTReportObject for now. I don't have a quarter million to spare ;) + +Example: + python vt_to_misp.py -i 719c97a8cd8db282586c1416894dcaf8 -c ./config.json + """ + parser = argparse.ArgumentParser(description=desc, epilog=post_desc, formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument("-e", "--event", help="MISP event id to add to") + parser.add_argument("-c", "--config", default="config.json", help="Path to JSON configuration file to read") + indicators = parser.add_mutually_exclusive_group(required=True) + indicators.add_argument("-i", "--indicator", help="Single indicator to look up") + indicators.add_argument("-f", "--file", help="File of indicators to look up - one on each line") + indicators.add_argument("-l", "--link", help="Link to a VirusTotal report") + return parser.parse_args() + + +def build_config(path=None): + ''' + Read a configuration file path. File is expected to be + + :path: Path to a configuration file + ''' + try: + with open(path, "r") as ifile: + return json.load(ifile) + except OSError: + raise OSError("Couldn't find path to configuration file: %s", path) + except json.JSONDecodeError: + raise IOError("Couldn't parse configuration file. Please make sure it is a proper JSON document") + + +def generate_report(indicator, apikey): + ''' + Build our VirusTotal report object, File object, and AV signature objects + and link them appropriately + + :indicator: Indicator hash to search in VT for + ''' + report_objects = [] + vt_report = VTReportObject(apikey, indicator) + report_objects.append(vt_report) + raw_report = vt_report._report + if vt_report._resource_type == "file": + file_object = pymisp.MISPObject(name="file") + file_object.add_attribute("md5", value=raw_report["md5"]) + file_object.add_attribute("sha1", value=raw_report["sha1"]) + file_object.add_attribute("sha256", value=raw_report["sha256"]) + vt_report.add_reference(referenced_uuid=file_object.uuid, relationship_type="report of") + report_objects.append(file_object) + elif vt_report._resource_type == "url": + parsed = urlsplit(indicator) + url_object = pymisp.MISPObject(name="url") + url_object.add_attribute("url", value=parsed.geturl()) + url_object.add_attribute("host", value=parsed.hostname) + url_object.add_attribute("scheme", value=parsed.scheme) + url_object.add_attribute("port", value=parsed.port) + vt_report.add_reference(referenced_uuid=url_object.uuid, relationship_type="report of") + report_objects.append(url_object) + for antivirus in raw_report["scans"]: + if raw_report["scans"][antivirus]["detected"]: + av_object = pymisp.MISPObject(name="av-signature") + av_object.add_attribute("software", value=antivirus) + signature_name = raw_report["scans"][antivirus]["result"] + av_object.add_attribute("signature", value=signature_name, disable_correlation=True) + vt_report.add_reference(referenced_uuid=av_object.uuid, relationship_type="included-in") + report_objects.append(av_object) + return report_objects + + +def get_misp_event(event_id=None, info=None): + ''' + Smaller helper function for generating a new MISP event or using a preexisting one + + :event_id: The event id of the MISP event to upload objects to + + :info: The event's title/info + ''' + if event_id: + event = misp.get_event(event_id) + elif info: + event = misp.new_event(info=info) + else: + event = misp.new_event(info="VirusTotal Report") + misp_event = pymisp.MISPEvent() + misp_event.load(event) + return misp_event + + +def main(misp, config, args): + ''' + Main program logic + + :misp: PyMISP API object for interfacing with MISP + + :config: Configuration dictionary + + :args: Argparse CLI object + ''' + if args.indicator: + misp_objects = generate_report(args.indicator, config["virustotal"]["key"]) + if misp_objects: + misp_event = get_misp_event(args.event, "VirusTotal Report for {}".format(args.indicator)) + submit_to_misp(misp, misp_event, misp_objects) + elif args.file: + try: + reports = [] + with open(args.file, "r") as ifile: + for indicator in ifile: + try: + misp_objects = generate_report(indicator, config["virustotal"]["key"]) + if misp_objects: + reports.append(misp_objects) + except pymisp.exceptions.InvalidMISPObject as err: + logging.error(err) + if reports: + current_time = datetime.now().strftime("%x %X") + misp_event = get_misp_event(args.event, "VirusTotal Reports: {}".format(current_time)) + for report in reports: + submit_to_misp(misp, misp_event, report) + except OSError: + logging.error("Couldn't open indicators file at '%s'. Check path", args.file) + elif args.link: + # https://www.virustotal.com/#/file//detection + indicator = args.link.split("/")[5] + misp_objects = generate_report(indicator, config["virustotal"]["key"]) + if misp_objects: + misp_event = get_misp_event(args.event, "VirusTotal Report for {}".format(indicator)) + submit_to_misp(misp, misp_event, misp_objects) + + +def submit_to_misp(misp, misp_event, misp_objects): + ''' + Submit a list of MISP objects to a MISP event + + :misp: PyMISP API object for interfacing with MISP + + :misp_event: MISPEvent object + + :misp_objects: List of MISPObject objects. Must be a list + ''' +# go through round one and only add MISP objects + for misp_object in misp_objects: + template_id = misp.get_object_template_id(misp_object.template_uuid) + misp.add_object(misp_event.id, template_id, misp_object) + # go through round two and add all the object references for each object + for misp_object in misp_objects: + for reference in misp_object.ObjectReference: + misp.add_object_reference(reference) + + +if __name__ == "__main__": + try: + args = build_cli() + config = build_config(args.config) + # change the 'ssl' value if you want to verify your MISP's SSL instance + misp = pymisp.PyMISP(url=config["misp"]["url"], key=config["misp"]["key"], ssl=False) + # finally, let's start checking VT and converting the reports + main(misp, config, args) + except KeyboardInterrupt: + print("Bye Felicia") + except pymisp.exceptions.InvalidMISPObject as err: + logging.error(err) diff --git a/pymisp/tools/__init__.py b/pymisp/tools/__init__.py index a0c667a..0412fde 100644 --- a/pymisp/tools/__init__.py +++ b/pymisp/tools/__init__.py @@ -1,3 +1,4 @@ +from .vtreportobject import VTReportObject # noqa from .neo4j import Neo4j # noqa from .fileobject import FileObject # noqa from .peobject import PEObject, PESectionObject # noqa diff --git a/pymisp/tools/vtreportobject.py b/pymisp/tools/vtreportobject.py new file mode 100644 index 0000000..5a4858b --- /dev/null +++ b/pymisp/tools/vtreportobject.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import re + +import requests +import validators + +from .abstractgenerator import AbstractMISPObjectGenerator +from .. import InvalidMISPObject + + +class VTReportObject(AbstractMISPObjectGenerator): + ''' + VirusTotal Report + + :apikey: VirusTotal API key (private works, but only public features are supported right now) + + :indicator: IOC to search VirusTotal for + ''' + def __init__(self, apikey, indicator): + # PY3 way: + # super().__init__("virustotal-report") + super(VTReportObject, self).__init__("virustotal-report") + indicator = indicator.strip() + self._resource_type = self.__validate_resource(indicator) + if self._resource_type: + self._report = self.__query_virustotal(apikey, indicator) + self.generate_attributes() + else: + error_msg = "A valid indicator is required. (One of type url, md5, sha1, sha256). Received '{}' instead".format(indicator) + raise InvalidMISPObject(error_msg) + # Mark as non_jsonable because we need to add the references manually after the object(s) have been created + self.update_not_jsonable('ObjectReference') + + def generate_attributes(self): + ''' Parse the VirusTotal report for relevant attributes ''' + self.add_attribute("last-submission", value=self._report["scan_date"]) + self.add_attribute("permalink", value=self._report["permalink"]) + ratio = "{}/{}".format(self._report["positives"], self._report["total"]) + self.add_attribute("detection-ratio", value=ratio) + + def __validate_resource(self, ioc): + ''' + Validate the data type of an indicator. + Domains and IP addresses aren't supported because + they don't return the same type of data as the URLs/files do + + :ioc: Indicator to search VirusTotal for + ''' + if validators.url(ioc): + return "url" + elif re.match(r"\b([a-fA-F0-9]{32}|[a-fA-F0-9]{40}|[a-fA-F0-9]{64})\b", ioc): + return "file" + return False + + def __query_virustotal(self, apikey, resource): + ''' + Query VirusTotal for information about an indicator + + :apikey: VirusTotal API key + + :resource: Indicator to search in VirusTotal + ''' + url = "https://www.virustotal.com/vtapi/v2/{}/report".format(self._resource_type) + params = {"apikey": apikey, "resource": resource} + # for now assume we're using a public API key - we'll figure out private keys later + report = requests.get(url, params=params) + report = report.json() + if report["response_code"] == 1: + return report + else: + error_msg = "{}: {}".format(resource, report["verbose_msg"]) + raise InvalidMISPObject(error_msg) From a8daa9b97268aece5ae90cd7255f379480d5f056 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 25 Oct 2017 11:17:25 -0400 Subject: [PATCH 22/36] Fix test suite --- tests/test_offline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_offline.py b/tests/test_offline.py index 47e43a5..70f8e3e 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -47,7 +47,7 @@ class TestOffline(unittest.TestCase): m.register_uri('POST', self.domain + 'events/5758ebf5-c898-48e6-9fe9-5665c0a83866', json=self.event) m.register_uri('DELETE', self.domain + 'events/2', json={'message': 'Event deleted.'}) m.register_uri('DELETE', self.domain + 'events/3', json={'errors': ['Invalid event'], 'message': 'Invalid event', 'name': 'Invalid event', 'url': '/events/3'}) - m.register_uri('DELETE', self.domain + 'attributes/2', json={'message': 'Attribute deleted.'}) + m.register_uri('GET', self.domain + 'attributes/delete/2', json={'message': 'Attribute deleted.'}) m.register_uri('POST', self.domain + 'events/index', json=self.search_index_result) def test_getEvent(self, m): From 78eb6e3080d1314bdaaa187a81f9b0e3d71f3dc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 25 Oct 2017 15:00:00 -0400 Subject: [PATCH 23/36] fix: Update dependencies for VT generator. --- .travis.yml | 2 +- pymisp/tools/vtreportobject.py | 9 ++++++++- setup.py | 3 ++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index e6a5ba7..93ef5eb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,7 @@ install: - pip install -U nose pip setuptools - pip install coveralls codecov requests-mock - pip install git+https://github.com/kbandla/pydeep.git - - pip install .[fileobjects,neo,openioc] + - pip install .[fileobjects,neo,openioc,virustotal] - pushd tests - git clone https://github.com/viper-framework/viper-test-files.git - popd diff --git a/pymisp/tools/vtreportobject.py b/pymisp/tools/vtreportobject.py index 5a4858b..927ca26 100644 --- a/pymisp/tools/vtreportobject.py +++ b/pymisp/tools/vtreportobject.py @@ -4,7 +4,12 @@ import re import requests -import validators +try: + import validators + has_validators = True +except ImportError: + has_validators = False + from .abstractgenerator import AbstractMISPObjectGenerator from .. import InvalidMISPObject @@ -48,6 +53,8 @@ class VTReportObject(AbstractMISPObjectGenerator): :ioc: Indicator to search VirusTotal for ''' + if not has_validators: + raise Exception('You need to install validators: pip install validators') if validators.url(ioc): return "url" elif re.match(r"\b([a-fA-F0-9]{32}|[a-fA-F0-9]{40}|[a-fA-F0-9]{64})\b", ioc): diff --git a/setup.py b/setup.py index 528b2d6..5d7391d 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,8 @@ setup( install_requires=['six', 'requests', 'python-dateutil', 'jsonschema', 'setuptools>=36.4'], extras_require={'fileobjects': ['lief>=0.8', 'python-magic'], 'neo': ['py2neo'], - 'openioc': ['beautifulsoup4']}, + 'openioc': ['beautifulsoup4'], + 'virustotal': ['validators']}, tests_require=[ 'jsonschema', 'python-dateutil', From 14bc9e4b19199b695c84e97a5baa7ea64bbfd6a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 26 Oct 2017 12:05:51 -0400 Subject: [PATCH 24/36] fix: Properly pop the distribution key. --- pymisp/mispevent.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index f00c5cf..3f99d7a 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -177,8 +177,9 @@ class MISPAttribute(AbstractMISP): if not isinstance(self.to_ids, bool): raise NewAttributeError('{} is invalid, to_ids has to be True or False'.format(self.to_ids)) - if kwargs.get('distribution') is not None: - self.distribution = int(kwargs.pop('distribution')) + self.distribution = kwargs.pop('distribution', None) + if self.distribution is not None: + self.distribution = int(self.distribution) if self.distribution not in [0, 1, 2, 3, 4, 5]: raise NewAttributeError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 4, 5'.format(self.distribution)) From 30da658292e9671f0c32bfe28bcebacb04b59540 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sat, 28 Oct 2017 16:57:03 -0400 Subject: [PATCH 25/36] chg: Remove warning if PyMISP is too new --- pymisp/api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index adc9784..fde42d8 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -118,9 +118,9 @@ class PyMISP(object): else: pymisp_version_tup = tuple(int(x) for x in __version__.split('.')) recommended_version_tup = tuple(int(x) for x in response['version'].split('.')) - if recommended_version_tup < pymisp_version_tup: - logger.warning("The version of PyMISP recommended by the MISP instance ({}) is older than the one you're using now ({}). Please upgrade the MISP instance or use an older PyMISP version.".format(response['version'], __version__)) - elif pymisp_version_tup < recommended_version_tup: + if recommended_version_tup < pymisp_version_tup[:3]: + logger.info("The version of PyMISP recommended by the MISP instance ({}) is older than the one you're using now ({}). If you have a problem, please upgrade the MISP instance or use an older PyMISP version.".format(response['version'], __version__)) + elif pymisp_version_tup[:3] < recommended_version_tup: logger.warning("The version of PyMISP recommended by the MISP instance ({}) is newer than the one you're using now ({}). Please upgrade PyMISP.".format(response['version'], __version__)) except Exception as e: From ea327ceffb1d19aeba027498270528e03b89b64c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sat, 28 Oct 2017 16:58:50 -0400 Subject: [PATCH 26/36] chg: Update asciidoctor generator --- examples/asciidoc_generator.py | 115 +++++++++++------------------ examples/profiles/__init__.py | 0 examples/profiles/daily_report.py | 37 ++++++++++ examples/profiles/weekly_report.py | 33 +++++++++ 4 files changed, 115 insertions(+), 70 deletions(-) create mode 100644 examples/profiles/__init__.py create mode 100644 examples/profiles/daily_report.py create mode 100644 examples/profiles/weekly_report.py diff --git a/examples/asciidoc_generator.py b/examples/asciidoc_generator.py index 156a7ff..f1a09a5 100755 --- a/examples/asciidoc_generator.py +++ b/examples/asciidoc_generator.py @@ -1,55 +1,21 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +import argparse +from datetime import date +import importlib + from pymisp import MISPEvent from defang import defang -import argparse from pytaxonomies import Taxonomies -from datetime import date -headers = """ -:toc: right -:toclevels: 1 -:toc-title: Daily Report -:icons: font -:sectanchors: -:sectlinks: -= Daily report by {org_name} -{date} - -:icons: font - -""" - -event_level_tags = """ -IMPORTANT: This event is classified TLP:{value}. - -{expanded} - -""" - -attributes = """ -=== Indicator(s) of compromise - -{list_attributes} - -""" - -title = """ -== ({internal_id}) {title} - -{summary} - -""" - -types_to_attach = ['ip-dst', 'url', 'domain'] -objects_to_attach = ['domain-ip'] class ReportGenerator(): - - def __init__(self): + def __init__(self, profile="daily_report"): self.taxonomies = Taxonomies() self.report = '' + profile_name = "profiles.{}".format(profile) + self.template = importlib.import_module(name=profile_name) def from_remote(self, event_id): from pymisp import PyMISP @@ -66,15 +32,16 @@ class ReportGenerator(): def attributes(self): if not self.misp_event.attributes: return '' - list_attributes = '' + list_attributes = [] for attribute in self.misp_event.attributes: - if attribute.type in types_to_attach: - list_attributes += "\n* {}\n".format(defang(attribute.value)) + if attribute.type in self.template.types_to_attach: + list_attributes.append("* {}".format(defang(attribute.value))) for obj in self.misp_event.Object: - for attribute in obj.Attribute: - if attribute.type in types_to_attach: - list_attributes += "\n* {}\n".format(defang(attribute.value)) - return attributes.format(list_attributes=list_attributes) + if obj.name in self.template.objects_to_attach: + for attribute in obj.Attribute: + if attribute.type in self.template.types_to_attach: + list_attributes.append("* {}".format(defang(attribute.value))) + return self.template.attributes.format(list_attributes="\n".join(list_attributes)) def _get_tag_info(self, machinetag): return self.taxonomies.revert_machinetag(machinetag) @@ -82,7 +49,7 @@ class ReportGenerator(): def report_headers(self): content = {'org_name': 'name', 'date': date.today().isoformat()} - self.report += headers.format(**content) + self.report += self.template.headers.format(**content) def event_level_tags(self): if not self.misp_event.Tag: @@ -91,7 +58,7 @@ class ReportGenerator(): # Only look for TLP for now if tag['name'].startswith('tlp'): tax, predicate = self._get_tag_info(tag['name']) - return event_level_tags.format(value=predicate.predicate.upper(), expanded=predicate.expanded) + return self.template.event_level_tags.format(value=predicate.predicate.upper(), expanded=predicate.expanded) def title(self): internal_id = '' @@ -106,34 +73,42 @@ class ReportGenerator(): if a.object_relation == 'summary': summary = a.value - return title.format(internal_id=internal_id, title=self.misp_event.info, - summary=summary) - + return self.template.title.format(internal_id=internal_id, title=self.misp_event.info, + summary=summary) def asciidoc(self, lang='en'): self.report += self.title() self.report += self.event_level_tags() self.report += self.attributes() + if __name__ == '__main__': + try: + parser = argparse.ArgumentParser(description='Create a human-readable report out of a MISP event') + parser.add_argument("--profile", default="daily_report", help="Profile template to use") + parser.add_argument("-o", "--output", help="Output file to write to (generally ends in .adoc)") + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("-e", "--event", default=[], nargs='+', help="Event ID to get.") + group.add_argument("-p", "--path", default=[], nargs='+', help="Path to the JSON dump.") - parser = argparse.ArgumentParser(description='Create a human-readable report out of a MISP event') - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument("-e", "--event", default=[], nargs='+', help="Event ID to get.") - group.add_argument("-p", "--path", default=[], nargs='+', help="Path to the JSON dump.") + args = parser.parse_args() - args = parser.parse_args() + report = ReportGenerator(args.profile) + report.report_headers() - report = ReportGenerator() - report.report_headers() + if args.event: + for eid in args.event: + report.from_remote(eid) + report.asciidoc() + else: + for f in args.path: + report.from_file(f) + report.asciidoc() - if args.event: - for eid in args.event: - report.from_remote(eid) - report.asciidoc() - else: - for f in args.path: - report.from_file(f) - report.asciidoc() - - print(report.report) + if args.output: + with open(args.output, "w") as ofile: + ofile.write(report.report) + else: + print(report.report) + except ModuleNotFoundError as err: + print(err) diff --git a/examples/profiles/__init__.py b/examples/profiles/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/examples/profiles/daily_report.py b/examples/profiles/daily_report.py new file mode 100644 index 0000000..2799333 --- /dev/null +++ b/examples/profiles/daily_report.py @@ -0,0 +1,37 @@ +types_to_attach = ['ip-dst', 'url', 'domain'] +objects_to_attach = ['domain-ip'] + +headers = """ +:toc: right +:toclevels: 1 +:toc-title: Daily Report +:icons: font +:sectanchors: +:sectlinks: += Daily report by {org_name} +{date} + +:icons: font + +""" + +event_level_tags = """ +IMPORTANT: This event is classified TLP:{value}. + +{expanded} + +""" + +attributes = """ +=== Indicator(s) of compromise + +{list_attributes} + +""" + +title = """ +== ({internal_id}) {title} + +{summary} + +""" diff --git a/examples/profiles/weekly_report.py b/examples/profiles/weekly_report.py new file mode 100644 index 0000000..1005fd9 --- /dev/null +++ b/examples/profiles/weekly_report.py @@ -0,0 +1,33 @@ +types_to_attach = ['ip-dst', 'url', 'domain', 'md5'] +objects_to_attach = ['domain-ip', 'file'] + +headers = """ +:toc: right +:toclevels: 1 +:toc-title: Weekly Report +:icons: font +:sectanchors: +:sectlinks: += Weekly report by {org_name} +{date} + +:icons: font + +""" + +event_level_tags = """ +""" + +attributes = """ +=== Indicator(s) of compromise + +{list_attributes} + +""" + +title = """ +== ({internal_id}) {title} + +{summary} + +""" From 7ece6b7fbcc2be9e645f52acd804abd63689410f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sat, 28 Oct 2017 17:09:11 -0400 Subject: [PATCH 27/36] fix: Properly set the distribution at event level fix #120 --- pymisp/api.py | 2 +- pymisp/mispevent.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index fde42d8..0069153 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -265,7 +265,7 @@ class PyMISP(object): misp_event.publish() return misp_event - def _prepare_full_attribute(self, category, type_value, value, to_ids, comment=None, distribution=5, **kwargs): + def _prepare_full_attribute(self, category, type_value, value, to_ids, comment=None, distribution=None, **kwargs): """Initialize a new MISPAttribute from scratch""" misp_attribute = MISPAttribute(self.describe_types) misp_attribute.set_all_values(type=type_value, value=value, category=category, diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 3f99d7a..130d905 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -428,8 +428,9 @@ class MISPEvent(AbstractMISP): raise NewAttributeError('The info field of the new event is required.') # Default values for a valid event to send to a MISP instance - if kwargs.get('distribution') is not None: - self.distribution = int(kwargs.pop('distribution')) + self.distribution = kwargs.pop('distribution', None) + if self.distribution is not None: + self.distribution = int(self.distribution) if self.distribution not in [0, 1, 2, 3, 4]: raise NewAttributeError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 4'.format(self.distribution)) From 4b53b399d0efb87ecb7a8f55ba78314ceeecc820 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 1 Nov 2017 17:15:49 -0700 Subject: [PATCH 28/36] fix: Properly upload a sample in an existing event. Fix https://github.com/MISP/PyMISP/issues/123 --- pymisp/api.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index 0069153..2272877 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -759,8 +759,6 @@ class PyMISP(object): to_post['request']['info'] = misp_event.info to_post['request']['analysis'] = misp_event.analysis to_post['request']['threat_level_id'] = misp_event.threat_level_id - else: - to_post['request']['event_id'] = int(event_id) default_values = self.sane_default['malware-sample'] if to_ids is None or not isinstance(to_ids, bool): @@ -772,7 +770,7 @@ class PyMISP(object): to_post['request']['category'] = category to_post['request']['comment'] = comment - return to_post + return to_post, event_id def _encode_file_to_upload(self, filepath_or_bytes): """Helper to encode a file to upload""" @@ -787,29 +785,33 @@ class PyMISP(object): to_ids=True, category=None, comment=None, info=None, analysis=None, threat_level_id=None): """Upload a sample""" - to_post = self._prepare_upload(event_id, distribution, to_ids, category, - comment, info, analysis, threat_level_id) + to_post, event_id = self._prepare_upload(event_id, distribution, to_ids, category, + comment, info, analysis, threat_level_id) to_post['request']['files'] = [{'filename': filename, 'data': self._encode_file_to_upload(filepath_or_bytes)}] - return self._upload_sample(to_post) + return self._upload_sample(to_post, event_id) def upload_samplelist(self, filepaths, event_id, distribution=None, to_ids=True, category=None, comment=None, info=None, analysis=None, threat_level_id=None): """Upload a list of samples""" - to_post = self._prepare_upload(event_id, distribution, to_ids, category, - comment, info, analysis, threat_level_id) + to_post, event_id = self._prepare_upload(event_id, distribution, to_ids, category, + comment, info, analysis, threat_level_id) files = [] for path in filepaths: if not os.path.isfile(path): continue files.append({'filename': os.path.basename(path), 'data': self._encode_file_to_upload(path)}) to_post['request']['files'] = files - return self._upload_sample(to_post) + return self._upload_sample(to_post, event_id) - def _upload_sample(self, to_post): + def _upload_sample(self, to_post, event_id=None): """Helper to upload a sample""" session = self.__prepare_session() - url = urljoin(self.root_url, 'events/upload_sample') + if event_id is None: + url = urljoin(self.root_url, 'events/upload_sample') + else: + url = urljoin(self.root_url, 'events/upload_sample/{}'.format(event_id)) + logger.info(to_post) response = session.post(url, data=json.dumps(to_post)) return self._check_response(response) From e5a7153284b3fdcba3b135e2785023194e95d677 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 1 Nov 2017 17:26:58 -0700 Subject: [PATCH 29/36] fix: Properly pass the distribution when uploading a sample Fix: https://github.com/MISP/PyMISP/issues/129 --- pymisp/api.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pymisp/api.py b/pymisp/api.py index 2272877..bbbfe84 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -759,6 +759,9 @@ class PyMISP(object): to_post['request']['info'] = misp_event.info to_post['request']['analysis'] = misp_event.analysis to_post['request']['threat_level_id'] = misp_event.threat_level_id + else: + if distribution is not None: + to_post['request']['distribution'] = distribution default_values = self.sane_default['malware-sample'] if to_ids is None or not isinstance(to_ids, bool): From 75d96b2d7a89406997035a1715de7c3cbce2303b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 2 Nov 2017 09:26:05 -0700 Subject: [PATCH 30/36] fix: Allow to load non-malware ZIP files in MISP Event Prior to his patch, any zip file loaded by MISP Event was unpacked and processed as an excrypted malware from MISP. --- pymisp/mispevent.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 130d905..098dd4f 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -221,16 +221,33 @@ class MISPAttribute(AbstractMISP): self._malware_binary = self.data self.encrypt = True + def __is_misp_encrypted_file(self, f): + files_list = f.namelist() + if len(files_list) != 2: + return False + md5_from_filename = '' + md5_from_file = '' + for name in files_list: + if name.endswith('.filename.txt'): + md5_from_filename = name.replace('.filename.txt', '') + else: + md5_from_file = name + if not md5_from_filename or not md5_from_file or md5_from_filename != md5_from_file: + return False + return True + def _load_data(self): if not isinstance(self.data, BytesIO): self.data = BytesIO(base64.b64decode(self.data)) if self.type == 'malware-sample': try: with ZipFile(self.data) as f: + if not self.__is_misp_encrypted_file(f): + raise Exception('Not an existing malware sample') for name in f.namelist(): - if name.endswith('.txt'): + if name.endswith('.filename.txt'): with f.open(name, pwd=b'infected') as unpacked: - self.malware_filename = unpacked.read().decode() + self.malware_filename = unpacked.read().decode().strip() else: with f.open(name, pwd=b'infected') as unpacked: self._malware_binary = BytesIO(unpacked.read()) From 0e123af546b7de00f65ee65a4b72402c3d10dd11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 2 Nov 2017 09:57:53 -0700 Subject: [PATCH 31/36] fix: Let load unknown object relations in known templates This isn't recommended, but happens very often. --- pymisp/mispevent.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 098dd4f..20eeb49 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -721,7 +721,11 @@ class MISPObject(AbstractMISP): if value.get('value') is None: return None if self.__known_template: - attribute = MISPObjectAttribute(self.__definition['attributes'][object_relation]) + if self.__definition['attributes'].get(object_relation): + attribute = MISPObjectAttribute(self.__definition['attributes'][object_relation]) + else: + # Woopsie, this object_relation is unknown, no sane defaults for you. + attribute = MISPObjectAttribute({}) else: attribute = MISPObjectAttribute({}) attribute.from_dict(object_relation, **value) From 134df0cafb8897da8798fb16e58ff4732c245f70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 7 Nov 2017 18:10:04 -0800 Subject: [PATCH 32/36] chg: Properly use python logging module. --- pymisp/__init__.py | 10 +++++-- pymisp/abstract.py | 6 +++-- pymisp/api.py | 42 ++++++++++++++---------------- pymisp/data/misp-objects | 2 +- pymisp/mispevent.py | 5 ++-- pymisp/tools/create_misp_object.py | 16 +++++++----- pymisp/tools/elfobject.py | 5 ++-- pymisp/tools/fileobject.py | 9 ++++--- pymisp/tools/machoobject.py | 6 +++-- pymisp/tools/peobject.py | 5 ++-- 10 files changed, 60 insertions(+), 46 deletions(-) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 55d2a19..3edf8b1 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,4 +1,9 @@ __version__ = '2.4.81.2' +import sys +import logging +logger = logging.getLogger(__name__) +FORMAT = "[%(filename)s:%(lineno)s - %(funcName)s() ] %(message)s" +logging.basicConfig(stream=sys.stderr, level=logging.WARNING, format=FORMAT) try: from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate # noqa @@ -9,5 +14,6 @@ try: from .tools import Neo4j # noqa from .tools import stix # noqa from .tools import openioc # noqa -except ImportError: - pass + logger.debug('pymisp loaded properly') +except ImportError as e: + logger.warning('Unable to load pymisp properly: {}'.format(e)) diff --git a/pymisp/abstract.py b/pymisp/abstract.py index a57795d..2a5d697 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -6,10 +6,12 @@ import json from json import JSONEncoder import collections import six # Remove that import when discarding python2 support. +import logging + +logger = logging.getLogger('pymisp') if six.PY2: - import warnings - warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5") + logger.warning("You're using python 2, it is strongly recommended to use python >=3.5") class MISPEncode(JSONEncoder): diff --git a/pymisp/api.py b/pymisp/api.py index bbbfe84..3ec4ed3 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -9,16 +9,19 @@ import datetime import os import base64 import re -import warnings import functools import logging +logger = logging.getLogger('pymisp') try: from urllib.parse import urljoin + # Least dirty way to support python 2 and 3 + basestring = str + unicode = str except ImportError: from urlparse import urljoin - warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5") + loger.warning("You're using python 2, it is strongly recommended to use python >=3.5") from io import BytesIO, open import zipfile @@ -40,19 +43,6 @@ from .exceptions import PyMISPError, SearchError, MissingDependency, NoURL, NoKe from .mispevent import MISPEvent, MISPAttribute from .abstract import MISPEncode -logger = logging.getLogger(__name__) - - -# Least dirty way to support python 2 and 3 -try: - basestring - unicode - warnings.warn("You're using python 2, it is strongly recommended to use python >=3.4") -except NameError: - basestring = str - unicode = str - - def deprecated(func): '''This is a decorator which can be used to mark functions as deprecated. It will result in a warning being emitted @@ -99,14 +89,14 @@ class PyMISP(object): self.cert = cert self.asynch = asynch if asynch and not ASYNC_OK: - warnings.warn("You turned on Async, but don't have requests_futures installed") + logger.warning("You turned on Async, but don't have requests_futures installed") self.asynch = False self.resources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') if out_type != 'json': raise PyMISPError('The only output type supported by PyMISP is JSON. If you still rely on XML, use PyMISP v2.4.49') if debug is not None: - warnings.warn('debug is deprecated, configure logging in api client') + logger.warning('debug is deprecated, configure logging in your script: import logging; logging.getLogger(\'pymisp\').setLevel(logging.DEBUG)') try: # Make sure the MISP instance is working and the URL is valid @@ -163,6 +153,8 @@ class PyMISP(object): 'Accept': 'application/{}'.format(output), 'content-type': 'application/{}'.format(output), 'User-Agent': 'PyMISP {} - Python {}.{}.{}'.format(__version__, *sys.version_info)}) + if logger.isEnabledFor(logging.DEBUG): + logger.debug(session.headers) return session # ##################### @@ -209,15 +201,16 @@ class PyMISP(object): def _check_response(self, response): """Check if the response from the server is not an unexpected error""" + errors = [] if response.status_code >= 500: - response.raise_for_status() + errors.append(response.json()) + logger.critical('Something bad happened on the server-side: {}'.format(response.json())) try: to_return = response.json() except ValueError: - logger.debug(response.text) - raise PyMISPError('Unknown error: {}'.format(response.text)) + # It the server didn't return a JSON blob, we've a problem. + raise PyMISPError('Unknown error (something is very broken server-side: {}'.format(response.text)) - errors = [] if isinstance(to_return, (list, str)): to_return = {'response': to_return} if to_return.get('error'): @@ -905,8 +898,9 @@ class PyMISP(object): if controller not in ['events', 'attributes']: raise Exception('Invalid controller. Can only be {}'.format(', '.join(['events', 'attributes']))) url = urljoin(self.root_url, '{}/{}'.format(controller, path.lstrip('/'))) - logger.debug('URL: %s', url) - logger.debug('Query: %s', query) + if logger.isEnabledFor(logging.DEBUG): + logger.debug('URL: %s', url) + logger.debug('Query: %s', query) if ASYNC_OK and isinstance(session, FuturesSession) and async_callback: response = session.post(url, data=json.dumps(query), background_callback=async_callback) @@ -1699,6 +1693,8 @@ class PyMISP(object): """Add an object""" session = self.__prepare_session() url = urljoin(self.root_url, 'objects/add/{}/{}'.format(event_id, template_id)) + if logger.isEnabledFor(logging.DEBUG): + logger.debug(url) response = session.post(url, data=misp_object.to_json()) return self._check_response(response) diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index bbf3e45..6b43b68 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit bbf3e45649af5af50c98ad90a86916cf75e8c74d +Subproject commit 6b43b68651a350a26891080ef0feda364b74727a diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 20eeb49..fc435fa 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -16,12 +16,13 @@ from collections import Counter from .abstract import AbstractMISP from .exceptions import UnknownMISPObjectTemplate, InvalidMISPObject, PyMISPError, NewEventError, NewAttributeError +import logging +logger = logging.getLogger('pymisp') import six # Remove that import when discarding python2 support. if six.PY2: - import warnings - warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5") + logger.warning("You're using python 2, it is strongly recommended to use python >=3.5") try: from dateutil.parser import parse diff --git a/pymisp/tools/create_misp_object.py b/pymisp/tools/create_misp_object.py index be16ae3..faa1258 100644 --- a/pymisp/tools/create_misp_object.py +++ b/pymisp/tools/create_misp_object.py @@ -3,7 +3,9 @@ from . import FileObject, PEObject, ELFObject, MachOObject from ..exceptions import MISPObjectException -import warnings +import logging + +logger = logging.getLogger('pymisp') try: import lief @@ -57,15 +59,15 @@ def make_binary_objects(filepath=None, pseudofile=None, filename=None): elif isinstance(lief_parsed, lief.MachO.Binary): return make_macho_objects(lief_parsed, misp_file) except lief.bad_format as e: - warnings.warn('\tBad format: {}'.format(e)) + logger.warning('Bad format: {}'.format(e)) except lief.bad_file as e: - warnings.warn('\tBad file: {}'.format(e)) + logger.warning('Bad file: {}'.format(e)) except lief.parser_error as e: - warnings.warn('\tParser error: {}'.format(e)) + logger.warning('Parser error: {}'.format(e)) except FileTypeNotImplemented as e: # noqa - warnings.warn(e) + logger.warning(e) if not HAS_LIEF: - warnings.warn('Please install lief, documentation here: https://github.com/lief-project/LIEF') + logger.warning('Please install lief, documentation here: https://github.com/lief-project/LIEF') if not filepath: - warnings.warn('LIEF currently requires a filepath and not a pseudo file') + logger.warning('LIEF currently requires a filepath and not a pseudo file') return misp_file, None, None diff --git a/pymisp/tools/elfobject.py b/pymisp/tools/elfobject.py index ee3bd29..d9c9561 100644 --- a/pymisp/tools/elfobject.py +++ b/pymisp/tools/elfobject.py @@ -5,8 +5,9 @@ from .abstractgenerator import AbstractMISPObjectGenerator from ..exceptions import InvalidMISPObject from io import BytesIO from hashlib import md5, sha1, sha256, sha512 -import warnings +import logging +logger = logging.getLogger('pymisp') try: import lief @@ -25,7 +26,7 @@ class ELFObject(AbstractMISPObjectGenerator): def __init__(self, parsed=None, filepath=None, pseudofile=None): if not HAS_PYDEEP: - warnings.warn("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") + logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") if not HAS_LIEF: raise ImportError('Please install lief, documentation here: https://github.com/lief-project/LIEF') if pseudofile: diff --git a/pymisp/tools/fileobject.py b/pymisp/tools/fileobject.py index e75884c..a12d38f 100644 --- a/pymisp/tools/fileobject.py +++ b/pymisp/tools/fileobject.py @@ -8,7 +8,10 @@ from io import BytesIO from hashlib import md5, sha1, sha256, sha512 import math from collections import Counter -import warnings +import logging + +logger = logging.getLogger('pymisp') + try: import pydeep @@ -27,9 +30,9 @@ class FileObject(AbstractMISPObjectGenerator): def __init__(self, filepath=None, pseudofile=None, filename=None): if not HAS_PYDEEP: - warnings.warn("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") + logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") if not HAS_MAGIC: - warnings.warn("Please install python-magic: pip install python-magic.") + logger.warning("Please install python-magic: pip install python-magic.") if filename: # Useful in case the file is copied with a pre-defined name by a script but we want to keep the original name self.__filename = filename diff --git a/pymisp/tools/machoobject.py b/pymisp/tools/machoobject.py index 15663c9..6cf3fa2 100644 --- a/pymisp/tools/machoobject.py +++ b/pymisp/tools/machoobject.py @@ -5,7 +5,9 @@ from ..exceptions import InvalidMISPObject from .abstractgenerator import AbstractMISPObjectGenerator from io import BytesIO from hashlib import md5, sha1, sha256, sha512 -import warnings +import logging + +logger = logging.getLogger('pymisp') try: @@ -25,7 +27,7 @@ class MachOObject(AbstractMISPObjectGenerator): def __init__(self, parsed=None, filepath=None, pseudofile=None): if not HAS_PYDEEP: - warnings.warn("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") + logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") if not HAS_LIEF: raise ImportError('Please install lief, documentation here: https://github.com/lief-project/LIEF') if pseudofile: diff --git a/pymisp/tools/peobject.py b/pymisp/tools/peobject.py index 3467243..2e8bf4a 100644 --- a/pymisp/tools/peobject.py +++ b/pymisp/tools/peobject.py @@ -6,8 +6,9 @@ from .abstractgenerator import AbstractMISPObjectGenerator from io import BytesIO from hashlib import md5, sha1, sha256, sha512 from datetime import datetime -import warnings +import logging +logger = logging.getLogger('pymisp') try: import lief @@ -26,7 +27,7 @@ class PEObject(AbstractMISPObjectGenerator): def __init__(self, parsed=None, filepath=None, pseudofile=None): if not HAS_PYDEEP: - warnings.warn("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") + logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") if not HAS_LIEF: raise ImportError('Please install lief, documentation here: https://github.com/lief-project/LIEF') if pseudofile: From f1a88f460e50ed8eca67b1b04ece7448f4c47705 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 7 Nov 2017 18:19:57 -0800 Subject: [PATCH 33/36] fix: Typo loger -> logger --- pymisp/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/api.py b/pymisp/api.py index 3ec4ed3..a7e993f 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -21,7 +21,7 @@ try: unicode = str except ImportError: from urlparse import urljoin - loger.warning("You're using python 2, it is strongly recommended to use python >=3.5") + logger.warning("You're using python 2, it is strongly recommended to use python >=3.5") from io import BytesIO, open import zipfile From 4512a4eaca040017facb133b78f84e80e265525d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 7 Nov 2017 19:10:54 -0800 Subject: [PATCH 34/36] chg: small improvments in the logging system --- pymisp/__init__.py | 2 +- pymisp/api.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 3edf8b1..ed4316b 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -2,7 +2,7 @@ __version__ = '2.4.81.2' import sys import logging logger = logging.getLogger(__name__) -FORMAT = "[%(filename)s:%(lineno)s - %(funcName)s() ] %(message)s" +FORMAT = "%(levelname)s [%(filename)s:%(lineno)s - %(funcName)s() ] %(message)s" logging.basicConfig(stream=sys.stderr, level=logging.WARNING, format=FORMAT) try: diff --git a/pymisp/api.py b/pymisp/api.py index a7e993f..0908936 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -95,8 +95,9 @@ class PyMISP(object): self.resources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') if out_type != 'json': raise PyMISPError('The only output type supported by PyMISP is JSON. If you still rely on XML, use PyMISP v2.4.49') - if debug is not None: - logger.warning('debug is deprecated, configure logging in your script: import logging; logging.getLogger(\'pymisp\').setLevel(logging.DEBUG)') + if debug: + logger.setLevel(logging.DEBUG) + logger.info('To configure logging in your script, leave it to None and use the following: import logging; logging.getLogger(\'pymisp\').setLevel(logging.DEBUG)') try: # Make sure the MISP instance is working and the URL is valid From f54a029e2a88566ee63011921a32d2d87bccca5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 8 Nov 2017 17:33:55 -0800 Subject: [PATCH 35/36] new: Proper debug system Make it easy to investigate the json blobs sent to the server. --- pymisp/abstract.py | 2 +- pymisp/api.py | 339 +++++++++++++++++-------------------------- pymisp/exceptions.py | 1 + pymisp/mispevent.py | 11 +- 4 files changed, 141 insertions(+), 212 deletions(-) diff --git a/pymisp/abstract.py b/pymisp/abstract.py index 2a5d697..151a3bd 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -64,7 +64,7 @@ class AbstractMISP(collections.MutableMapping): return self.to_dict() def to_json(self): - return json.dumps(self.to_dict(), cls=MISPEncode) + return json.dumps(self, cls=MISPEncode) def __getitem__(self, key): return getattr(self, key) diff --git a/pymisp/api.py b/pymisp/api.py index 0908936..839fe73 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -11,6 +11,14 @@ import base64 import re import functools import logging +import warnings +from io import BytesIO, open +import zipfile + +from . import __version__ +from .exceptions import PyMISPError, SearchError, NoURL, NoKey +from .mispevent import MISPEvent, MISPAttribute +from .abstract import MISPEncode logger = logging.getLogger('pymisp') @@ -22,8 +30,6 @@ try: except ImportError: from urlparse import urljoin logger.warning("You're using python 2, it is strongly recommended to use python >=3.5") -from io import BytesIO, open -import zipfile try: import requests @@ -38,10 +44,6 @@ try: except ImportError: ASYNC_OK = False -from . import __version__ -from .exceptions import PyMISPError, SearchError, MissingDependency, NoURL, NoKey -from .mispevent import MISPEvent, MISPAttribute -from .abstract import MISPEncode def deprecated(func): '''This is a decorator which can be used to mark functions @@ -89,7 +91,7 @@ class PyMISP(object): self.cert = cert self.asynch = asynch if asynch and not ASYNC_OK: - logger.warning("You turned on Async, but don't have requests_futures installed") + logger.critical("You turned on Async, but don't have requests_futures installed") self.asynch = False self.resources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') @@ -118,8 +120,7 @@ class PyMISP(object): raise PyMISPError('Unable to connect to MISP ({}). Please make sure the API key and the URL are correct (http/https is required): {}'.format(self.root_url, e)) try: - session = self.__prepare_session() - response = session.get(urljoin(self.root_url, 'attributes/describeTypes.json')) + response = self.__prepare_request('GET', urljoin(self.root_url, 'attributes/describeTypes.json')) describe_types = self._check_response(response) if describe_types.get('error'): for e in describe_types.get('error'): @@ -137,26 +138,32 @@ class PyMISP(object): self.category_type_mapping = self.describe_types['category_type_mappings'] self.sane_default = self.describe_types['sane_defaults'] - def __prepare_session(self, output='json', async_implemented=False): - """Prepare the session headers""" - - if not HAVE_REQUESTS: - raise MissingDependency('Missing dependency, install requests (`pip install requests`)') - if self.asynch and async_implemented: - session = FuturesSession() + def __prepare_request(self, request_type, url, data=None, + background_callback=None, output_type='json'): + if logger.isEnabledFor(logging.DEBUG): + logger.debug('{} - {}'.format(request_type, url)) + if data is not None: + logger.debug(data) + if data is None: + req = requests.Request(request_type, url) else: - session = requests.Session() - session.verify = self.ssl - session.proxies = self.proxies - session.cert = self.cert - session.headers.update( + req = requests.Request(request_type, url, data=data) + if self.asynch and background_callback is not None: + s = FuturesSession() + else: + s = requests.Session() + prepped = s.prepare_request(req) + prepped.headers.update( {'Authorization': self.key, - 'Accept': 'application/{}'.format(output), - 'content-type': 'application/{}'.format(output), + 'Accept': 'application/{}'.format(output_type), + 'content-type': 'application/{}'.format(output_type), 'User-Agent': 'PyMISP {} - Python {}.{}.{}'.format(__version__, *sys.version_info)}) if logger.isEnabledFor(logging.DEBUG): - logger.debug(session.headers) - return session + logger.debug(prepped.headers) + if self.asynch and background_callback is not None: + return s.send(prepped, verify=self.ssl, proxies=self.proxies, cert=self.cert, background_callback=background_callback) + else: + return s.send(prepped, verify=self.ssl, proxies=self.proxies, cert=self.cert) # ##################### # ### Core helpers #### @@ -294,13 +301,11 @@ class PyMISP(object): Warning, there's a limit on the number of results """ - session = self.__prepare_session() url = urljoin(self.root_url, 'events/index') - if filters is not None: - filters = json.dumps(filters) - response = session.post(url, data=filters) + if filters is None: + response = self.__prepare_request('GET', url) else: - response = session.get(url) + response = self.__prepare_request('POST', url, json.dumps(filters)) return self._check_response(response) def get_event(self, event_id): @@ -308,9 +313,8 @@ class PyMISP(object): :param event_id: Event id to get """ - session = self.__prepare_session() url = urljoin(self.root_url, 'events/{}'.format(event_id)) - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def add_event(self, event): @@ -318,14 +322,12 @@ class PyMISP(object): :param event: Event as JSON object / string to add """ - session = self.__prepare_session() url = urljoin(self.root_url, 'events') if isinstance(event, MISPEvent): - event = json.dumps(event, cls=MISPEncode) - if isinstance(event, basestring): - response = session.post(url, data=event) - else: - response = session.post(url, data=json.dumps(event)) + event = event.to_json() + elif not isinstance(event, basestring): + event = json.dumps(event) + response = self.__prepare_request('POST', url, event) return self._check_response(response) def update_event(self, event_id, event): @@ -334,14 +336,12 @@ class PyMISP(object): :param event_id: Event id to update :param event: Event as JSON object / string to add """ - session = self.__prepare_session() url = urljoin(self.root_url, 'events/{}'.format(event_id)) if isinstance(event, MISPEvent): - event = json.dumps(event, cls=MISPEncode) - if isinstance(event, basestring): - response = session.post(url, data=event) - else: - response = session.post(url, data=json.dumps(event)) + event = event.to_json() + elif not isinstance(event, basestring): + event = json.dumps(event) + response = self.__prepare_request('POST', url, event) return self._check_response(response) def delete_event(self, event_id): @@ -349,26 +349,23 @@ class PyMISP(object): :param event_id: Event id to delete """ - session = self.__prepare_session() url = urljoin(self.root_url, 'events/{}'.format(event_id)) - response = session.delete(url) + response = self.__prepare_request('DELETE', url) return self._check_response(response) def delete_attribute(self, attribute_id, hard_delete=False): """Delete an attribute by ID""" - session = self.__prepare_session() if hard_delete: url = urljoin(self.root_url, 'attributes/delete/{}/1'.format(attribute_id)) else: url = urljoin(self.root_url, 'attributes/delete/{}'.format(attribute_id)) - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def pushEventToZMQ(self, event_id): """Force push an event on ZMQ""" - session = self.__prepare_session() url = urljoin(self.root_url, 'events/pushEventToZMQ/{}.json'.format(event_id)) - response = session.post(url) + response = self.__prepare_request('POST', url) return self._check_response(response) # ############################################## @@ -401,12 +398,11 @@ class PyMISP(object): event_id = full_event.id if full_event.published: return {'error': 'Already published'} - session = self.__prepare_session() if not alert: url = urljoin(self.root_url, 'events/publish/{}'.format(event_id)) else: url = urljoin(self.root_url, 'events/alert/{}'.format(event_id)) - response = session.post(url) + response = self.__prepare_request('POST', url) return self._check_response(response) def change_threat_level(self, event, threat_level_id): @@ -431,20 +427,18 @@ class PyMISP(object): """Tag an event or an attribute""" if not self._valid_uuid(uuid): raise PyMISPError('Invalid UUID') - session = self.__prepare_session() + url = urljoin(self.root_url, 'tags/attachTagToObject') to_post = {'uuid': uuid, 'tag': tag} - path = 'tags/attachTagToObject' - response = session.post(urljoin(self.root_url, path), data=json.dumps(to_post)) + response = self.__prepare_request('POST', url, json.dumps(to_post)) return self._check_response(response) def untag(self, uuid, tag): """Untag an event or an attribute""" if not self._valid_uuid(uuid): raise PyMISPError('Invalid UUID') - session = self.__prepare_session() + url = urljoin(self.root_url, 'tags/removeTagFromObject') to_post = {'uuid': uuid, 'tag': tag} - path = 'tags/removeTagFromObject' - response = session.post(urljoin(self.root_url, path), data=json.dumps(to_post)) + response = self.__prepare_request('POST', url, json.dumps(to_post)) return self._check_response(response) # ##### File attributes ##### @@ -474,9 +468,8 @@ class PyMISP(object): if proposal: response = self.proposal_add(eventID_to_update, a) else: - session = self.__prepare_session() url = urljoin(self.root_url, 'attributes/add/{}'.format(eventID_to_update)) - response = self._check_response(session.post(url, data=json.dumps(a, cls=MISPEncode))) + response = self.__prepare_request('POST', url, a.to_json()) return response def add_named_attribute(self, event, type_value, value, category=None, to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): @@ -803,61 +796,54 @@ class PyMISP(object): def _upload_sample(self, to_post, event_id=None): """Helper to upload a sample""" - session = self.__prepare_session() if event_id is None: url = urljoin(self.root_url, 'events/upload_sample') else: url = urljoin(self.root_url, 'events/upload_sample/{}'.format(event_id)) - logger.info(to_post) - response = session.post(url, data=json.dumps(to_post)) + response = self.__prepare_request('POST', url, json.dumps(to_post)) return self._check_response(response) # ############################ # ######## Proposals ######### # ############################ - def __query_proposal(self, session, path, id, attribute=None): + def __query_proposal(self, path, id, attribute=None): """Helper to prepare a query to handle proposals""" url = urljoin(self.root_url, 'shadow_attributes/{}/{}'.format(path, id)) if path in ['add', 'edit']: query = {'request': {'ShadowAttribute': attribute}} - response = session.post(url, data=json.dumps(query, cls=MISPEncode)) + response = self.__prepare_request('POST', url, json.dumps(query, cls=MISPEncode)) elif path == 'view': - response = session.get(url) + response = self.__prepare_request('GET', url) else: # accept or discard - response = session.post(url) + response = self.__prepare_request('POST', url) return self._check_response(response) def proposal_view(self, event_id=None, proposal_id=None): """View a proposal""" - session = self.__prepare_session() if proposal_id is not None and event_id is not None: return {'error': 'You can only view an event ID or a proposal ID'} if event_id is not None: id = event_id else: id = proposal_id - return self.__query_proposal(session, 'view', id) + return self.__query_proposal('view', id) def proposal_add(self, event_id, attribute): """Add a proposal""" - session = self.__prepare_session() - return self.__query_proposal(session, 'add', event_id, attribute) + return self.__query_proposal('add', event_id, attribute) def proposal_edit(self, attribute_id, attribute): """Edit a proposal""" - session = self.__prepare_session() - return self.__query_proposal(session, 'edit', attribute_id, attribute) + return self.__query_proposal('edit', attribute_id, attribute) def proposal_accept(self, proposal_id): """Accept a proposal""" - session = self.__prepare_session() - return self.__query_proposal(session, 'accept', proposal_id) + return self.__query_proposal('accept', proposal_id) def proposal_discard(self, proposal_id): """Discard a proposal""" - session = self.__prepare_session() - return self.__query_proposal(session, 'discard', proposal_id) + return self.__query_proposal('discard', proposal_id) # ############################## # ###### Attribute update ###### @@ -868,8 +854,7 @@ class PyMISP(object): if to_ids not in [0, 1]: raise Exception('to_ids can only be 0 or 1') query = {"to_ids": to_ids} - session = self.__prepare_session() - return self.__query(session, 'edit/{}'.format(attribute_uuid), query, controller='attributes') + return self.__query('edit/{}'.format(attribute_uuid), query, controller='attributes') # ############################## # ###### Attribute update ###### @@ -885,28 +870,24 @@ class PyMISP(object): query['adhereToWarninglists'] = adhereToWarninglists if distribution is not None: query['distribution'] = distribution - session = self.__prepare_session() - return self.__query(session, 'freeTextImport/{}'.format(event_id), query, controller='events') + return self.__query('freeTextImport/{}'.format(event_id), query, controller='events') # ############################## # ######## REST Search ######### # ############################## - def __query(self, session, path, query, controller='events', async_callback=None): + def __query(self, path, query, controller='events', async_callback=None): """Helper to prepare a search query""" if query.get('error') is not None: return query if controller not in ['events', 'attributes']: raise Exception('Invalid controller. Can only be {}'.format(', '.join(['events', 'attributes']))) url = urljoin(self.root_url, '{}/{}'.format(controller, path.lstrip('/'))) - if logger.isEnabledFor(logging.DEBUG): - logger.debug('URL: %s', url) - logger.debug('Query: %s', query) - if ASYNC_OK and isinstance(session, FuturesSession) and async_callback: - response = session.post(url, data=json.dumps(query), background_callback=async_callback) + if ASYNC_OK and async_callback: + response = self.__prepare_request('POST', url, json.dumps(query), async_callback) else: - response = session.post(url, data=json.dumps(query)) + response = self.__prepare_request('POST', url, json.dumps(query)) return self._check_response(response) def search_index(self, published=None, eventid=None, tag=None, datefrom=None, @@ -952,13 +933,12 @@ class PyMISP(object): if not set(param).issubset(rule_levels[rule]): raise SearchError('Values in your {} are invalid, has to be in {}'.format(rule, ', '.join(str(x) for x in rule_levels[rule]))) to_post[rule] = '|'.join(str(x) for x in param) - session = self.__prepare_session(async_implemented=(async_callback is not None)) url = urljoin(self.root_url, buildup_url) if self.asynch and async_callback: - response = session.post(url, data=json.dumps(to_post), background_callback=async_callback) + response = self.__prepare_request('POST', url, json.dumps(to_post), async_callback) else: - response = session.post(url, data=json.dumps(to_post)) + response = self.__prepare_request('POST', url, json.dumps(to_post)) res = self._check_response(response) if normalize: to_return = {'response': []} @@ -971,8 +951,7 @@ class PyMISP(object): def search_all(self, value): """Search a value in the whole database""" query = {'value': value, 'searchall': 1} - session = self.__prepare_session() - return self.__query(session, 'restSearch/download', query) + return self.__query('restSearch/download', query) def __prepare_rest_search(self, values, not_values): """Prepare a search, generate the chain processed by the server @@ -1082,8 +1061,7 @@ class PyMISP(object): raise SearchError('Unused parameter: {}'.format(', '.join(kwargs.keys()))) # Create a session, make it async if and only if we have a callback - session = self.__prepare_session(async_implemented=(async_callback is not None)) - return self.__query(session, 'restSearch/download', query, controller, async_callback) + return self.__query('restSearch/download', query, controller, async_callback) def get_attachment(self, attribute_id): """Get an attachement (not a malware sample) by attribute ID. @@ -1091,9 +1069,8 @@ class PyMISP(object): :param attribute_id: Attribute ID to fetched """ - attach = urljoin(self.root_url, 'attributes/downloadAttachment/download/{}'.format(attribute_id)) - session = self.__prepare_session() - response = session.get(attach) + url = urljoin(self.root_url, 'attributes/downloadAttachment/download/{}'.format(attribute_id)) + response = self.__prepare_request('GET', url) try: response.json() # The query fails, response contains a json blob @@ -1104,9 +1081,9 @@ class PyMISP(object): def get_yara(self, event_id): """Get the yara rules from an event""" + url = urljoin(self.root_url, 'attributes/restSearch') to_post = {'request': {'eventid': event_id, 'type': 'yara'}} - session = self.__prepare_session() - response = session.post(urljoin(self.root_url, 'attributes/restSearch'), data=json.dumps(to_post)) + response = self.__prepare_request('POST', url, data=json.dumps(to_post)) result = self._check_response(response) if result.get('error') is not None: return False, result.get('error') @@ -1117,9 +1094,9 @@ class PyMISP(object): def download_samples(self, sample_hash=None, event_id=None, all_samples=False): """Download samples, by hash or event ID. If there are multiple samples in one event, use the all_samples switch""" + url = urljoin(self.root_url, 'attributes/downloadSample') to_post = {'request': {'hash': sample_hash, 'eventID': event_id, 'allSamples': all_samples}} - session = self.__prepare_session() - response = session.post(urljoin(self.root_url, 'attributes/downloadSample'), data=json.dumps(to_post)) + response = self.__prepare_request('POST', url, data=json.dumps(to_post)) result = self._check_response(response) if result.get('error') is not None: return False, result.get('error') @@ -1155,9 +1132,8 @@ class PyMISP(object): def get_all_tags(self, quiet=False): """Get all the tags used on the instance""" - session = self.__prepare_session() url = urljoin(self.root_url, 'tags') - response = session.get(url) + response = self.__prepare_request('GET', url) r = self._check_response(response) if not quiet or r.get('errors'): return r @@ -1170,9 +1146,8 @@ class PyMISP(object): def new_tag(self, name=None, colour="#00ace6", exportable=False): """Create a new tag""" to_post = {'Tag': {'name': name, 'colour': colour, 'exportable': exportable}} - session = self.__prepare_session() url = urljoin(self.root_url, 'tags/add') - response = session.post(url, data=json.dumps(to_post)) + response = self.__prepare_request('POST', url, json.dumps(to_post)) return self._check_response(response) # ########## Version ########## @@ -1192,16 +1167,14 @@ class PyMISP(object): def get_recommended_api_version(self): """Returns the recommended API version from the server""" - session = self.__prepare_session() url = urljoin(self.root_url, 'servers/getPyMISPVersion.json') - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def get_version(self): """Returns the version of the instance.""" - session = self.__prepare_session() url = urljoin(self.root_url, 'servers/getVersion.json') - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def get_version_master(self): @@ -1217,19 +1190,17 @@ class PyMISP(object): def get_attributes_statistics(self, context='type', percentage=None): """Get attributes statistics from the MISP instance""" - session = self.__prepare_session() if (context != 'category'): context = 'type' if percentage is not None: url = urljoin(self.root_url, 'attributes/attributeStatistics/{}/{}'.format(context, percentage)) else: url = urljoin(self.root_url, 'attributes/attributeStatistics/{}'.format(context)) - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def get_tags_statistics(self, percentage=None, name_sort=None): """Get tags statistics from the MISP instance""" - session = self.__prepare_session() if percentage is not None: percentage = 'true' else: @@ -1239,32 +1210,29 @@ class PyMISP(object): else: name_sort = 'false' url = urljoin(self.root_url, 'tags/tagStatistics/{}/{}'.format(percentage, name_sort)) - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) # ############## Sightings ################## def sighting_per_id(self, attribute_id): """Add a sighting to an attribute (by attribute ID)""" - session = self.__prepare_session() url = urljoin(self.root_url, 'sightings/add/{}'.format(attribute_id)) - response = session.post(url) + response = self.__prepare_request('POST', url) return self._check_response(response) def sighting_per_uuid(self, attribute_uuid): """Add a sighting to an attribute (by attribute UUID)""" - session = self.__prepare_session() url = urljoin(self.root_url, 'sightings/add/{}'.format(attribute_uuid)) - response = session.post(url) + response = self.__prepare_request('POST', url) return self._check_response(response) def set_sightings(self, sightings): """Push a sighting (python dictionary)""" if isinstance(sightings, dict): sightings = json.dumps(sightings) - session = self.__prepare_session() url = urljoin(self.root_url, 'sightings/add/') - response = session.post(url, data=sightings) + response = self.__prepare_request('POST', url, sightings) return self._check_response(response) def sighting_per_json(self, json_file): @@ -1277,9 +1245,8 @@ class PyMISP(object): def get_sharing_groups(self): """Get the existing sharing groups""" - session = self.__prepare_session() url = urljoin(self.root_url, 'sharing_groups.json') - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response)['response'] # ############## Users ################## @@ -1325,57 +1292,49 @@ class PyMISP(object): return user def get_users_list(self): - session = self.__prepare_session() url = urljoin(self.root_url, 'admin/users') - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response)['response'] def get_user(self, user_id): - session = self.__prepare_session() url = urljoin(self.root_url, 'admin/users/view/{}'.format(user_id)) - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def add_user(self, email, org_id, role_id, **kwargs): - new_user = self._set_user_parameters(**dict(email=email, org_id=org_id, role_id=role_id, **kwargs)) - session = self.__prepare_session() url = urljoin(self.root_url, 'admin/users/add/') - response = session.post(url, data=json.dumps(new_user)) + new_user = self._set_user_parameters(**dict(email=email, org_id=org_id, role_id=role_id, **kwargs)) + response = self.__prepare_request('POST', url, json.dumps(new_user)) return self._check_response(response) def add_user_json(self, json_file): - session = self.__prepare_session() with open(json_file, 'r') as f: jdata = json.load(f) url = urljoin(self.root_url, 'admin/users/add/') - response = session.post(url, data=json.dumps(jdata)) + response = self.__prepare_request('POST', url, json.dumps(jdata)) return self._check_response(response) def get_user_fields_list(self): - session = self.__prepare_session() url = urljoin(self.root_url, 'admin/users/add/') - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def edit_user(self, user_id, **kwargs): edit_user = self._set_user_parameters(**kwargs) - session = self.__prepare_session() url = urljoin(self.root_url, 'admin/users/edit/{}'.format(user_id)) - response = session.post(url, data=json.dumps(edit_user)) + response = self.__prepare_request('POST', url, json.dumps(edit_user)) return self._check_response(response) def edit_user_json(self, json_file, user_id): - session = self.__prepare_session() with open(json_file, 'r') as f: jdata = json.load(f) url = urljoin(self.root_url, 'admin/users/edit/{}'.format(user_id)) - response = session.post(url, data=json.dumps(jdata)) + response = self.__prepare_request('POST', url, json.dumps(jdata)) return self._check_response(response) def delete_user(self, user_id): - session = self.__prepare_session() url = urljoin(self.root_url, 'admin/users/delete/{}'.format(user_id)) - response = session.post(url) + response = self.__prepare_request('POST', url) return self._check_response(response) # ############## Organisations ################## @@ -1401,64 +1360,56 @@ class PyMISP(object): return organisation def get_organisations_list(self, scope="local"): - session = self.__prepare_session() scope = scope.lower() if scope not in ["local", "external", "all"]: raise ValueError("Authorized fields are 'local','external' or 'all'") url = urljoin(self.root_url, 'organisations/index/scope:{}'.format(scope)) - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response)['response'] def get_organisation(self, organisation_id): - session = self.__prepare_session() url = urljoin(self.root_url, 'organisations/view/{}'.format(organisation_id)) - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def add_organisation(self, name, **kwargs): new_org = self._set_organisation_parameters(**dict(name=name, **kwargs)) - session = self.__prepare_session() if 'local' in new_org: if new_org.get('local') is False: if 'uuid' not in new_org: raise PyMISPError('A remote org MUST have a valid uuid') url = urljoin(self.root_url, 'admin/organisations/add/') - response = session.post(url, data=json.dumps(new_org)) + response = self.__prepare_request('POST', url, json.dumps(new_org)) return self._check_response(response) def add_organisation_json(self, json_file): - session = self.__prepare_session() with open(json_file, 'r') as f: jdata = json.load(f) url = urljoin(self.root_url, 'admin/organisations/add/') - response = session.post(url, data=json.dumps(jdata)) + response = self.__prepare_request('POST', url, json.dumps(jdata)) return self._check_response(response) def get_organisation_fields_list(self): - session = self.__prepare_session() url = urljoin(self.root_url, 'admin/organisations/add/') - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def edit_organisation(self, org_id, **kwargs): edit_org = self._set_organisation_parameters(**kwargs) - session = self.__prepare_session() url = urljoin(self.root_url, 'admin/organisations/edit/{}'.format(org_id)) - response = session.post(url, data=json.dumps(edit_org)) + response = self.__prepare_request('POST', url, json.dumps(edit_org)) return self._check_response(response) def edit_organisation_json(self, json_file, org_id): - session = self.__prepare_session() with open(json_file, 'r') as f: jdata = json.load(f) url = urljoin(self.root_url, 'admin/organisations/edit/{}'.format(org_id)) - response = session.post(url, data=json.dumps(jdata)) + response = self.__prepare_request('POST', url, json.dumps(jdata)) return self._check_response(response) def delete_organisation(self, org_id): - session = self.__prepare_session() url = urljoin(self.root_url, 'admin/organisations/delete/{}'.format(org_id)) - response = session.post(url) + response = self.__prepare_request('POST', url) return self._check_response(response) # ############## Servers ################## @@ -1521,17 +1472,15 @@ class PyMISP(object): new_server = self._set_server_parameters(url, name, authkey, organisation, internal, push, pull, self_signed, push_rules, pull_rules, submitted_cert, submitted_client_cert, None, None) - session = self.__prepare_session() url = urljoin(self.root_url, 'servers/add') - response = session.post(url, data=json.dumps(new_server)) + response = self.__prepare_request('POST', url, json.dumps(new_server)) return self._check_response(response) def add_server_json(self, json_file): - session = self.__prepare_session() with open(json_file, 'r') as f: jdata = json.load(f) url = urljoin(self.root_url, 'servers/add') - response = session.post(url, data=json.dumps(jdata)) + response = self.__prepare_request('POST', url, json.dumps(jdata)) return self._check_response(response) def edit_server(self, server_id, url=None, name=None, authkey=None, organisation=None, internal=None, push=False, @@ -1540,35 +1489,31 @@ class PyMISP(object): new_server = self._set_server_parameters(url, name, authkey, organisation, internal, push, pull, self_signed, push_rules, pull_rules, submitted_cert, submitted_client_cert, delete_cert, delete_client_cert) - session = self.__prepare_session() url = urljoin(self.root_url, 'servers/edit/{}'.format(server_id)) - response = session.post(url, data=json.dumps(new_server)) + response = self.__prepare_request('POST', url, json.dumps(new_server)) return self._check_response(response) def edit_server_json(self, json_file, server_id): - session = self.__prepare_session() with open(json_file, 'r') as f: jdata = json.load(f) url = urljoin(self.root_url, 'servers/edit/{}'.format(server_id)) - response = session.post(url, data=json.dumps(jdata)) + response = self.__prepare_request('POST', url, json.dumps(jdata)) return self._check_response(response) # ############## Roles ################## def get_roles_list(self): """Get the list of existing roles""" - session = self.__prepare_session() url = urljoin(self.root_url, '/roles') - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response)['response'] # ############## Tags ################## def get_tags_list(self): """Get the list of existing tags""" - session = self.__prepare_session() url = urljoin(self.root_url, '/tags') - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response)['Tag'] # ############################################## @@ -1579,9 +1524,8 @@ class PyMISP(object): def download_all_suricata(self): """Download all suricata rules events.""" - suricata_rules = urljoin(self.root_url, 'events/nids/suricata/download') - session = self.__prepare_session('rules') - response = session.get(suricata_rules) + url = urljoin(self.root_url, 'events/nids/suricata/download') + response = self.__prepare_request('GET', url, output_type='rules') return response def download_suricata_rule_event(self, event_id): @@ -1589,18 +1533,16 @@ class PyMISP(object): :param event_id: ID of the event to download (same as get) """ - template = urljoin(self.root_url, 'events/nids/suricata/download/{}'.format(event_id)) - session = self.__prepare_session('rules') - response = session.get(template) + url = urljoin(self.root_url, 'events/nids/suricata/download/{}'.format(event_id)) + response = self.__prepare_request('GET', url, output_type='rules') return response # ############## Text ############### def get_all_attributes_txt(self, type_attr, tags=False, eventId=False, allowNonIDS=False, date_from=False, date_to=False, last=False, enforceWarninglist=False, allowNotPublished=False): """Get all attributes from a specific type as plain text. Only published and IDS flagged attributes are exported, except if stated otherwise.""" - session = self.__prepare_session('txt') url = urljoin(self.root_url, 'attributes/text/download/%s/%s/%s/%s/%s/%s/%s/%s/%s' % (type_attr, tags, eventId, allowNonIDS, date_from, date_to, last, enforceWarninglist, allowNotPublished)) - response = session.get(url) + response = self.__prepare_request('GET', url, output_type='txt') return response # ############## STIX ############## @@ -1610,12 +1552,10 @@ class PyMISP(object): if tags: if isinstance(tags, list): tags = "&&".join(tags) - - session = self.__prepare_session() url = urljoin(self.root_url, "/events/stix/download/{}/{}/{}/{}/{}".format( event_id, with_attachments, tags, from_date, to_date)) logger.debug("Getting STIX event from %s", url) - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def get_stix(self, **kwargs): @@ -1627,58 +1567,50 @@ class PyMISP(object): def fetch_feed(self, feed_id): """Fetch one single feed""" - session = self.__prepare_session() url = urljoin(self.root_url, 'feeds/fetchFromFeed/{}'.format(feed_id)) - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def view_feeds(self): """Get the content of all the feeds""" - session = self.__prepare_session() url = urljoin(self.root_url, 'feeds') - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def view_feed(self, feed_ids): """Get the content of a single feed""" - session = self.__prepare_session() url = urljoin(self.root_url, 'feeds/view/{}'.format(feed_ids)) - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def cache_feeds_all(self): """ Cache all the feeds""" - session = self.__prepare_session() url = urljoin(self.root_url, 'feeds/cacheFeeds/all') - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def cache_feed(self, feed_id): """Cache a specific feed""" - session = self.__prepare_session() url = urljoin(self.root_url, 'feeds/cacheFeeds/{}'.format(feed_id)) - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def cache_feeds_freetext(self): """Cache all the freetext feeds""" - session = self.__prepare_session() url = urljoin(self.root_url, 'feeds/cacheFeeds/freetext') - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def cache_feeds_misp(self): """Cache all the MISP feeds""" - session = self.__prepare_session() url = urljoin(self.root_url, 'feeds/cacheFeeds/misp') - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def compare_feeds(self): """Generate the comparison matrix for all the MISP feeds""" - session = self.__prepare_session() url = urljoin(self.root_url, 'feeds/compareFeeds') - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response) def cache_all_feeds(self): @@ -1692,25 +1624,20 @@ class PyMISP(object): def add_object(self, event_id, template_id, misp_object): """Add an object""" - session = self.__prepare_session() url = urljoin(self.root_url, 'objects/add/{}/{}'.format(event_id, template_id)) - if logger.isEnabledFor(logging.DEBUG): - logger.debug(url) - response = session.post(url, data=misp_object.to_json()) + response = self.__prepare_request('POST', url, misp_object.to_json()) return self._check_response(response) def add_object_reference(self, misp_object_reference): """Add a reference to an object""" - session = self.__prepare_session() url = urljoin(self.root_url, 'object_references/add') - response = session.post(url, data=misp_object_reference.to_json()) + response = self.__prepare_request('POST', url, misp_object_reference.to_json()) return self._check_response(response) def get_object_templates_list(self): """Returns the list of Object templates available on the MISP instance""" - session = self.__prepare_session() url = urljoin(self.root_url, 'objectTemplates') - response = session.get(url) + response = self.__prepare_request('GET', url) return self._check_response(response)['response'] def get_object_template_id(self, object_uuid): @@ -1727,7 +1654,6 @@ class PyMISP(object): @deprecated def add_tag(self, event, tag, attribute=False): - session = self.__prepare_session() if attribute: to_post = {'request': {'Attribute': {'id': event['id'], 'tag': tag}}} path = 'attributes/addTag' @@ -1737,17 +1663,18 @@ class PyMISP(object): event = event["Event"] to_post = {'request': {'Event': {'id': event['id'], 'tag': tag}}} path = 'events/addTag' - response = session.post(urljoin(self.root_url, path), data=json.dumps(to_post)) + url = urljoin(self.root_url, path) + response = self.__prepare_request('POST', url, json.dumps(to_post)) return self._check_response(response) @deprecated def remove_tag(self, event, tag, attribute=False): - session = self.__prepare_session() if attribute: to_post = {'request': {'Attribute': {'id': event['id'], 'tag': tag}}} path = 'attributes/removeTag' else: to_post = {'request': {'Event': {'id': event['Event']['id'], 'tag': tag}}} path = 'events/removeTag' - response = session.post(urljoin(self.root_url, path), data=json.dumps(to_post)) + url = urljoin(self.root_url, path) + response = self.__prepare_request('POST', url, json.dumps(to_post)) return self._check_response(response) diff --git a/pymisp/exceptions.py b/pymisp/exceptions.py index 8a09c10..d828e74 100644 --- a/pymisp/exceptions.py +++ b/pymisp/exceptions.py @@ -40,6 +40,7 @@ class InvalidMISPObject(MISPObjectException): """Exception raised when an object doesn't respect the contrains in the definition""" pass + class UnknownMISPObjectTemplate(MISPObjectException): """Exception raised when the template is unknown""" pass diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index fc435fa..645dc9b 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -16,10 +16,11 @@ from collections import Counter from .abstract import AbstractMISP from .exceptions import UnknownMISPObjectTemplate, InvalidMISPObject, PyMISPError, NewEventError, NewAttributeError +import six # Remove that import when discarding python2 support. + import logging logger = logging.getLogger('pymisp') -import six # Remove that import when discarding python2 support. if six.PY2: logger.warning("You're using python 2, it is strongly recommended to use python >=3.5") @@ -139,7 +140,7 @@ class MISPAttribute(AbstractMISP): try: c.verify(signed_data, signature=base64.b64decode(self.sig), verify=keys[:1]) return {self.uuid: True} - except: + except Exception: return {self.uuid: False} def set_all_values(self, **kwargs): @@ -252,7 +253,7 @@ class MISPAttribute(AbstractMISP): else: with f.open(name, pwd=b'infected') as unpacked: self._malware_binary = BytesIO(unpacked.read()) - except: + except Exception: # not a encrypted zip file, assuming it is a new malware sample self._prepare_new_malware_sample() @@ -383,7 +384,7 @@ class MISPEvent(AbstractMISP): try: c.verify(signed_data, signature=base64.b64decode(self.sig), verify=keys[:1]) to_return[self.uuid] = True - except: + except Exception: to_return[self.uuid] = False for a in self.attributes: to_return.update(a.verify(gpg_uid)) @@ -393,7 +394,7 @@ class MISPEvent(AbstractMISP): try: c.verify(to_verify_global, signature=base64.b64decode(self.global_sig), verify=keys[:1]) to_return['global'] = True - except: + except Exception: to_return['global'] = False return to_return From e6ab90012d5c5e1fd1c0973360ac87f18c0a742c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 8 Nov 2017 18:01:27 -0800 Subject: [PATCH 36/36] chg: Update readme for new logging system --- README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/README.md b/README.md index b8a77c0..de40a54 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,23 @@ cd examples python3 last.py -l 10 ``` +## Debugging + +You have two options there: + +1. Pass `debug=True` to `PyMISP` and it will enable logging.DEBUG to stderr on the whole module + +2. Use the python logging module directly: + +```python + +import logging +logger = logging.getLogger('pymisp') + +# Configure it as you whish, for example, enable DEBUG mode: +logger.setLevel(logging.DEBUG) +``` + ## Documentation [PyMISP API documentation is available](https://media.readthedocs.org/pdf/pymisp/master/pymisp.pdf).