From c8e9aa47d5b6da42de69f6b8385d719939dc7f78 Mon Sep 17 00:00:00 2001 From: Koen Van Impe Date: Tue, 24 Sep 2019 20:59:46 +0200 Subject: [PATCH 1/6] Disable to_ids based on false positive sightings reporting --- examples/falsepositive_disabletoids.py | 136 +++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100755 examples/falsepositive_disabletoids.py diff --git a/examples/falsepositive_disabletoids.py b/examples/falsepositive_disabletoids.py new file mode 100755 index 0000000..0280da6 --- /dev/null +++ b/examples/falsepositive_disabletoids.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +''' +Koen Van Impe + +Disable the to_ids flag of an attribute when there are to many false positives +Put this script in crontab to run every /15 or /60 + */5 * * * * mispuser /usr/bin/python3 /home/mispuser/PyMISP/examples/falsepositive_disabletoids.py + +Do inline config in "main" + +''' + +from pymisp import ExpandedPyMISP, MISPEvent +from keys import misp_url, misp_key, misp_verifycert +from datetime import datetime +from datetime import date + +import datetime as dt +import smtplib +import mimetypes +from email.mime.multipart import MIMEMultipart +from email import encoders +from email.mime.base import MIMEBase +from email.mime.text import MIMEText +import argparse + + +def init(url, key, verifycert): + ''' + Template to get MISP module started + ''' + return ExpandedPyMISP(url, key, verifycert, 'json') + + +if __name__ == '__main__': + + minimal_fp = 0 + threshold_to_ids = .50 + minimal_date_sighting_date = '1970-01-01 00:00:00' + + smtp_from = 'INSERT_FROM' + smtp_to = 'INSERT_TO' + smtp_server = 'localhost' + report_changes = '' + ts_format = '%Y-%m-%d %H:%M:%S' + + parser = argparse.ArgumentParser(description="Disable the to_ids flag of attributes with a certain number of false positives above a threshold.") + parser.add_argument('-m', '--mail', action='store_true', help='Mail the report') + parser.add_argument('-o', '--mailoptions', action='store', help='mailoptions: \'smtp_from=INSERT_FROM;smtp_to=INSERT_TO;smtp_server=localhost\'') + parser.add_argument('-b', '--minimal-fp', default=minimal_fp, type=int, help='Minimal number of false positive (default: %(default)s )') + parser.add_argument('-t', '--threshold', default=threshold_to_ids, type=float, help='Threshold false positive/true positive rate (default: %(default)s )') + parser.add_argument('-d', '--minimal-date-sighting', default=minimal_date_sighting_date, help='Minimal date for sighting (false positive / true positive) (default: %(default)s )') + + args = parser.parse_args() + misp = init(misp_url, misp_key, misp_verifycert) + + minimal_fp = int(args.minimal_fp) + threshold_to_ids = args.threshold + minimal_date_sighting_date = args.minimal_date_sighting + minimal_date_sighting = int(dt.datetime.strptime(minimal_date_sighting_date, '%Y-%m-%d %H:%M:%S').strftime("%s")) + + # Fetch all the attributes + result = misp.search('attributes', to_ids=1, include_sightings=1) + + if 'Attribute' in result: + for attribute in result['Attribute']: + true_positive = 0 + false_positive = 0 + compute_threshold = 0 + attribute_id = attribute['id'] + attribute_value = attribute['value'] + attribute_uuid = attribute['uuid'] + event_id = attribute['event_id'] + + # Only do something if there is a sighting + if 'Sighting' in attribute and len(attribute['Sighting']) > 0: + + for sighting in attribute['Sighting']: + if int(sighting['date_sighting']) > minimal_date_sighting: + if int(sighting['type']) == 0: + true_positive = true_positive + 1 + elif int(sighting['type']) == 1: + false_positive = false_positive + 1 + + if false_positive > minimal_fp: + compute_threshold = false_positive / (true_positive + false_positive) + + if compute_threshold >= threshold_to_ids: + # Fetch event title for report text + event_details = misp.get_event(event_id) + event_info = event_details['Event']['info'] + + misp.update_attribute( { 'uuid': attribute_uuid, 'to_ids': 0}) + + report_changes = report_changes + 'Disable to_ids for [%s] (%s) in event [%s] (%s) - FP: %s TP: %s \n' % (attribute_value, attribute_id, event_info, event_id, false_positive, true_positive) + + # Changing the attribute to_ids flag sets the event to unpublished + misp.publish(event_id) + + # Only send/print the report if it contains content + if len(report_changes) > 0: + if args.mail: + if args.mailoptions: + mailoptions = args.mailoptions.split(';') + for s in mailoptions: + if s.split('=')[0] == 'smtp_from': + smtp_from = s.split('=')[1] + if s.split('=')[0] == 'smtp_to': + smtp_to = s.split('=')[1] + if s.split('=')[0] == 'smtp_server': + smtp_server = s.split('=')[1] + + now = datetime.now() + current_date = now.strftime(ts_format) + report_changes_body = 'MISP Disable to_ids flags for %s on %s\n-------------------------------------------------------------------------------\n\n' % (misp_url, current_date) + report_changes_body = report_changes_body + 'Minimal number of false positives before considering threshold: %s\n' % (minimal_fp) + report_changes_body = report_changes_body + 'Threshold false positives/true positives to disable to_ids flag: %s\n' % (threshold_to_ids) + report_changes_body = report_changes_body + 'Minimal date for sighting false positives: %s\n\n' % (minimal_date_sighting_date) + report_changes_body = report_changes_body + report_changes + report_changes_body = report_changes_body + '\nEvents that have attributes with changed to_ids flag have been republished, without e-mail notification.' + report_changes_body = report_changes_body + '\n\nMISP Disable to_ids Finished\n' + + subject = 'Report of disable to_ids flag for false positives sightings of %s' % (current_date) + msg = MIMEMultipart() + msg['From'] = smtp_from + msg['To'] = smtp_to + msg['Subject'] = subject + + msg.attach(MIMEText(report_changes_body, 'text')) + print(report_changes_body) + server = smtplib.SMTP(smtp_server) + server.sendmail(smtp_from, smtp_to, msg.as_string()) + + else: + print(report_changes) From 8d81f318cdca854109579ca0a7de45a115b9b59c Mon Sep 17 00:00:00 2001 From: Antoine Cailliau Date: Thu, 26 Sep 2019 10:26:59 +0200 Subject: [PATCH 2/6] Adds support to add local tags. Requires https://github.com/MISP/MISP/pull/5215 to be merged first. --- pymisp/aping.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymisp/aping.py b/pymisp/aping.py index f810525..b3953b0 100644 --- a/pymisp/aping.py +++ b/pymisp/aping.py @@ -1952,13 +1952,13 @@ class ExpandedPyMISP(PyMISP): raise PyMISPError('The misp_entity must be MISPEvent, MISPObject or MISPAttribute') - def tag(self, misp_entity: Union[AbstractMISP, str], tag: str): + def tag(self, misp_entity: Union[AbstractMISP, str], tag: str, local: bool=False): """Tag an event or an attribute. misp_entity can be a UUID""" if 'uuid' in misp_entity: uuid = misp_entity.uuid else: uuid = misp_entity - to_post = {'uuid': uuid, 'tag': tag} + to_post = {'uuid': uuid, 'tag': tag, 'local': local} response = self._prepare_request('POST', 'tags/attachTagToObject', data=to_post) return self._check_response(response, expect_json=True) From ddb3c2e4ac78ce99b521730e978a6015d5a5a756 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Thu, 26 Sep 2019 11:03:03 +0200 Subject: [PATCH 3/6] Minor grammar errors --- docs/tutorial/FullOverview.ipynb | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/tutorial/FullOverview.ipynb b/docs/tutorial/FullOverview.ipynb index 5083657..783fa75 100644 --- a/docs/tutorial/FullOverview.ipynb +++ b/docs/tutorial/FullOverview.ipynb @@ -59,11 +59,11 @@ "source": [ "# Using the PyMISP objects\n", "\n", - "This page aims to give recommandations about how to efficiently use the `pymisp` library.\n", + "This page aims to give recommendations about how to efficiently use the `pymisp` library.\n", "\n", "It is strongly recommended (read \"don't do anything else, please\") to use the library this way and never, ever modify the python dictionary you get by loading the json blob you receive from the server.\n", "\n", - "This library is made in a way to hide as much as the complexity as possible and we're happy to improve it is there is someting missing." + "This library is made in a way to hide as much as the complexity as possible and we're happy to improve it is there is something missing." ] }, { @@ -121,7 +121,7 @@ "## Set the Event date\n", "\n", "\n", - "The date can be in many different formats. This helper makes sure it normalises it in a way that will be understood by your MISP instance." + "The date can be in many different formats. This helper makes sure it normalizes it in a way that will be understood by your MISP instance." ] }, { @@ -145,7 +145,7 @@ "event.set_date(d)\n", "print(event.date)\n", "\n", - "# datetime.datetime => MISP expects a day, so the hour will be droped.\n", + "# datetime.datetime => MISP expects a day, so the hour will be dropped.\n", "from datetime import datetime\n", "d = datetime.now()\n", "print(type(d))\n", @@ -159,7 +159,7 @@ "source": [ "## Add Attribute to event\n", "\n", - "More usefull things: adding attributes to an event.\n", + "More useful things: adding attributes to an event.\n", "\n", "Attributes have a bunch of parameters you can pass (if you feel like it). If you don't pass them, they'll be automatically set depending on their sane defaults.\n", "\n", @@ -263,7 +263,7 @@ "source": [ "## Soft delete attribute\n", "\n", - "**Important note**: the default approach to *delete* on MISP is to do a soft delete (meaning the attribue is not displayed on the default view on MISP). The reason we do it this way is that it allows to push *delete* updates to instances we synchronize with.\n", + "**Important note**: the default approach to *delete* on MISP is to do a soft delete (meaning the attribute is not displayed on the default view on MISP). The reason we do it this way is that it allows to push *delete* updates to instances we synchronize with.\n", "\n", "The delete method will set the default parameter of the attribute to `True`." ] @@ -533,7 +533,7 @@ "source": [ "## Generic helper\n", "\n", - "This helper is meant to be used when you alreadu have a script that does the mapping between your own code, and the MISPObject template." + "This helper is meant to be used when you already have a script that does the mapping between your own code, and the MISPObject template." ] }, { @@ -746,7 +746,7 @@ "source": [ "## Edit, removes the timestamp when exporting\n", "\n", - "If you tried to edit an event manually, and never got the updates on the instance, it is probably because the timestamps weren't updated/removed. Or you removed them all, and adding a single tag was makting every attributes as new.\n", + "If you tried to edit an event manually, and never got the updates on the instance, it is probably because the timestamps weren't updated/removed. Or you removed them all, and adding a single tag was making every attributes as new.\n", "\n", "PyMISP got you covered." ] @@ -803,7 +803,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Getting the API key (automatically generated on the trainig VM)" + "# Getting the API key (automatically generated on the training VM)" ] }, { From edaae39bc829c0a1ba7432a375a600de1393c3b5 Mon Sep 17 00:00:00 2001 From: Koen Van Impe Date: Thu, 26 Sep 2019 20:31:05 +0200 Subject: [PATCH 4/6] List all the sightings - show_sightings.py --- examples/show_sightings.py | 164 +++++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 examples/show_sightings.py diff --git a/examples/show_sightings.py b/examples/show_sightings.py new file mode 100644 index 0000000..bd8fdbc --- /dev/null +++ b/examples/show_sightings.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +''' +Koen Van Impe + +List all the sightings + +Put this script in crontab to run every day + 25 4 * * * mispuser /usr/bin/python3 /home/mispuser/PyMISP/examples/show_sightings.py + +''' + +from pymisp import ExpandedPyMISP +from keys import misp_url, misp_key, misp_verifycert + +import sys +import time +from datetime import datetime +import smtplib +import mimetypes +from email.mime.multipart import MIMEMultipart +from email import encoders +from email.mime.base import MIMEBase +from email.mime.text import MIMEText +import argparse + + +def init(url, key, verifycert): + ''' + Template to get MISP module started + ''' + return ExpandedPyMISP(url, key, verifycert, 'json') + + +def set_drift_timestamp(drift_timestamp, drift_timestamp_path): + ''' + Save the timestamp in a (local) file + ''' + try: + with open(drift_timestamp_path, 'w+') as f: + f.write(str(drift_timestamp)) + return True + except IOError: + sys.exit("Unable to write drift_timestamp %s to %s" % (drift_timestamp, drift_timestamp_path)) + return False + + +def get_drift_timestamp(drift_timestamp_path): + ''' + From when do we start with the sightings? + ''' + try: + with open(drift_timestamp_path) as f: + drift = f.read() + if drift: + drift = int(float(drift)) + else: + drift = 0 + except IOError: + drift = 0 + + return drift + + +def search_sightings(misp, from_timestamp, end_timestamp): + ''' + Search all the sightings + ''' + completed_sightings = [] + + try: + found_sightings = misp.search_sightings(date_from=from_timestamp, date_to=end_timestamp) + except Exception as e: + sys.exit('Unable to search for sightings') + + if found_sightings is not None: + for s in found_sightings: + if 'Sighting' in s: + sighting = s['Sighting'] + if 'attribute_id' in sighting: + attribute_id = sighting['attribute_id'] + + # Query the attribute and event to get the details + try: + attribute = misp.get_attribute(attribute_id) + except Exception as e: + continue + + if 'Attribute' in attribute and 'uuid' in attribute['Attribute']: + event_details = misp.get_event(attribute['Attribute']['event_id']) + event_info = event_details['Event']['info'] + attribute_uuid = attribute['Attribute']['uuid'] + completed_sightings.append({'attribute_uuid': attribute_uuid, 'date_sighting': sighting['date_sighting'], 'source': sighting['source'], 'type': sighting['type'], 'uuid': sighting['uuid'], 'event_id': attribute['Attribute']['event_id'], 'value': attribute['Attribute']['value'], 'attribute_id': attribute['Attribute']['id'], 'event_title': event_info}) + else: + continue + + return completed_sightings + + +if __name__ == '__main__': + smtp_from = 'INSERT_FROM' + smtp_to = 'INSERT_TO' + smtp_server = 'localhost' + report_sightings = '' + ts_format = '%Y-%m-%d %H:%M:%S' + drift_timestamp_path = '/home/mispuser/PyMISP/examples/show_sightings.drift' + + parser = argparse.ArgumentParser(description="Show all the sightings.") + parser.add_argument('-m', '--mail', action='store_true', help='Mail the report') + parser.add_argument('-o', '--mailoptions', action='store', help='mailoptions: \'smtp_from=INSERT_FROM;smtp_to=INSERT_TO;smtp_server=localhost\'') + + args = parser.parse_args() + misp = init(misp_url, misp_key, misp_verifycert) + + start_timestamp = get_drift_timestamp(drift_timestamp_path=drift_timestamp_path) + end_timestamp = time.time() + start_timestamp_s = datetime.fromtimestamp(start_timestamp).strftime(ts_format) + end_timestamp_s = datetime.fromtimestamp(end_timestamp).strftime(ts_format) + + # Get all attribute sightings + found_sightings = search_sightings(misp, start_timestamp, end_timestamp) + if found_sightings is not None and len(found_sightings) > 0: + for s in found_sightings: + if int(s['type']) == 0: + type = 'TP' + else: + type = 'FP' + date_sighting = datetime.fromtimestamp(int(s['date_sighting'])).strftime(ts_format) + source = s['source'] + if not s['source']: + source = 'N/A' + report_sightings = report_sightings + '%s for [%s] (%s) in event [%s] (%s) on %s from %s\n' % (type, s['value'], s['attribute_id'], s['event_title'], s['event_id'], date_sighting, source) + + set_drift_timestamp(end_timestamp, drift_timestamp_path) + else: + report_sightings = 'No sightings found' + + # Mail options + if args.mail: + if args.mailoptions: + mailoptions = args.mailoptions.split(';') + for s in mailoptions: + if s.split('=')[0] == 'smtp_from': + smtp_from = s.split('=')[1] + if s.split('=')[0] == 'smtp_to': + smtp_to = s.split('=')[1] + if s.split('=')[0] == 'smtp_server': + smtp_server = s.split('=')[1] + + report_sightings_body = 'MISP Sightings report for %s between %s and %s\n-------------------------------------------------------------------------------\n\n' % (misp_url, start_timestamp_s, end_timestamp_s) + report_sightings_body = report_sightings_body + report_sightings + subject = 'Report of sightings between %s and %s' % (start_timestamp_s, end_timestamp_s) + + msg = MIMEMultipart() + msg['From'] = smtp_from + msg['To'] = smtp_to + msg['Subject'] = subject + + msg.attach(MIMEText(report_sightings_body, 'text')) + server = smtplib.SMTP(smtp_server) + server.sendmail(smtp_from, smtp_to, msg.as_string()) + + else: + print(report_sightings) From 0e68071ef285d831104d3202e5c4e7bddb18f59b Mon Sep 17 00:00:00 2001 From: Koen Van Impe Date: Thu, 26 Sep 2019 20:46:31 +0200 Subject: [PATCH 5/6] Update type and code cleanup --- examples/show_sightings.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/examples/show_sightings.py b/examples/show_sightings.py index bd8fdbc..5e83bcc 100644 --- a/examples/show_sightings.py +++ b/examples/show_sightings.py @@ -84,6 +84,7 @@ def search_sightings(misp, from_timestamp, end_timestamp): try: attribute = misp.get_attribute(attribute_id) except Exception as e: + print("Unable to fetch attribute") continue if 'Attribute' in attribute and 'uuid' in attribute['Attribute']: @@ -119,17 +120,17 @@ if __name__ == '__main__': # Get all attribute sightings found_sightings = search_sightings(misp, start_timestamp, end_timestamp) - if found_sightings is not None and len(found_sightings) > 0: + if found_sightings: for s in found_sightings: if int(s['type']) == 0: - type = 'TP' + s_type = 'TP' else: - type = 'FP' + s_type = 'FP' date_sighting = datetime.fromtimestamp(int(s['date_sighting'])).strftime(ts_format) source = s['source'] if not s['source']: source = 'N/A' - report_sightings = report_sightings + '%s for [%s] (%s) in event [%s] (%s) on %s from %s\n' % (type, s['value'], s['attribute_id'], s['event_title'], s['event_id'], date_sighting, source) + report_sightings = report_sightings + '%s for [%s] (%s) in event [%s] (%s) on %s from %s\n' % (s_type, s['value'], s['attribute_id'], s['event_title'], s['event_id'], date_sighting, source) set_drift_timestamp(end_timestamp, drift_timestamp_path) else: From 5b7eeaa8ab19642884327961272cf5bac824d5bc Mon Sep 17 00:00:00 2001 From: Koen Van Impe Date: Thu, 26 Sep 2019 20:50:53 +0200 Subject: [PATCH 6/6] Code cleanup --- examples/falsepositive_disabletoids.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/falsepositive_disabletoids.py b/examples/falsepositive_disabletoids.py index 0280da6..c4a72d1 100755 --- a/examples/falsepositive_disabletoids.py +++ b/examples/falsepositive_disabletoids.py @@ -74,7 +74,7 @@ if __name__ == '__main__': event_id = attribute['event_id'] # Only do something if there is a sighting - if 'Sighting' in attribute and len(attribute['Sighting']) > 0: + if 'Sighting' in attribute: for sighting in attribute['Sighting']: if int(sighting['date_sighting']) > minimal_date_sighting: @@ -99,7 +99,7 @@ if __name__ == '__main__': misp.publish(event_id) # Only send/print the report if it contains content - if len(report_changes) > 0: + if report_changes: if args.mail: if args.mailoptions: mailoptions = args.mailoptions.split(';')