dos2unix examples/stats_report.py

pull/557/head
Sebastian Wagner 2020-03-17 15:45:07 +01:00
parent 3136b44204
commit 240b1e1617
No known key found for this signature in database
GPG Key ID: 4BB5A67FBECFA8A5
1 changed files with 405 additions and 405 deletions

View File

@ -1,405 +1,405 @@
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
''' '''
Koen Van Impe Koen Van Impe
Maxime Thiebaut Maxime Thiebaut
Generate a report of your MISP statistics Generate a report of your MISP statistics
Put this script in crontab to run every /15 or /60 Put this script in crontab to run every /15 or /60
*/5 * * * * mispuser /usr/bin/python3 /home/mispuser/PyMISP/examples/stats_report.py -t 30d -m -v */5 * * * * mispuser /usr/bin/python3 /home/mispuser/PyMISP/examples/stats_report.py -t 30d -m -v
Do inline config in "main" Do inline config in "main"
''' '''
from pymisp import ExpandedPyMISP from pymisp import ExpandedPyMISP
from keys import misp_url, misp_key, misp_verifycert from keys import misp_url, misp_key, misp_verifycert
import argparse import argparse
import os import os
from datetime import datetime from datetime import datetime
from datetime import date from datetime import date
import time import time
import sys import sys
import smtplib import smtplib
import mimetypes import mimetypes
from email.mime.multipart import MIMEMultipart from email.mime.multipart import MIMEMultipart
from email import encoders from email import encoders
from email.mime.base import MIMEBase from email.mime.base import MIMEBase
from email.mime.text import MIMEText from email.mime.text import MIMEText
# Suppress those "Unverified HTTPS request is being made" # Suppress those "Unverified HTTPS request is being made"
import urllib3 import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
def init(url, key, verifycert): def init(url, key, verifycert):
''' '''
Template to get MISP module started Template to get MISP module started
''' '''
return ExpandedPyMISP(url, key, verifycert, 'json') return ExpandedPyMISP(url, key, verifycert, 'json')
def get_data(misp, timeframe, date_from=None, date_to=None): def get_data(misp, timeframe, date_from=None, date_to=None):
''' '''
Get the event date to build our report Get the event date to build our report
''' '''
number_of_misp_events = 0 number_of_misp_events = 0
number_of_attributes = 0 number_of_attributes = 0
number_of_attributes_to_ids = 0 number_of_attributes_to_ids = 0
attr_type = {} attr_type = {}
attr_category = {} attr_category = {}
tags_type = {} tags_type = {}
tags_tlp = {'tlp:white': 0, 'tlp:green': 0, 'tlp:amber': 0, 'tlp:red': 0} tags_tlp = {'tlp:white': 0, 'tlp:green': 0, 'tlp:amber': 0, 'tlp:red': 0}
tags_misp_galaxy_mitre = {} tags_misp_galaxy_mitre = {}
tags_misp_galaxy = {} tags_misp_galaxy = {}
tags_misp_galaxy_threat_actor = {} tags_misp_galaxy_threat_actor = {}
galaxies = {} galaxies = {}
galaxies_cluster = {} galaxies_cluster = {}
threat_levels_counts = [0, 0, 0, 0] threat_levels_counts = [0, 0, 0, 0]
analysis_completion_counts = [0, 0, 0] analysis_completion_counts = [0, 0, 0]
report = {} report = {}
try: try:
if date_from and date_to: if date_from and date_to:
stats_event_response = misp.search(date_from=date_from, date_to=date_to) stats_event_response = misp.search(date_from=date_from, date_to=date_to)
else: else:
stats_event_response = misp.search(last=timeframe) stats_event_response = misp.search(last=timeframe)
# Number of new or updated events since timestamp # Number of new or updated events since timestamp
report['number_of_misp_events'] = len(stats_event_response) report['number_of_misp_events'] = len(stats_event_response)
report['misp_events'] = [] report['misp_events'] = []
for event in stats_event_response: for event in stats_event_response:
event_data = event['Event'] event_data = event['Event']
timestamp = datetime.utcfromtimestamp(int(event_data['timestamp'])).strftime(ts_format) timestamp = datetime.utcfromtimestamp(int(event_data['timestamp'])).strftime(ts_format)
publish_timestamp = datetime.utcfromtimestamp(int(event_data['publish_timestamp'])).strftime(ts_format) publish_timestamp = datetime.utcfromtimestamp(int(event_data['publish_timestamp'])).strftime(ts_format)
threat_level_id = int(event_data['threat_level_id']) - 1 threat_level_id = int(event_data['threat_level_id']) - 1
threat_levels_counts[threat_level_id] = threat_levels_counts[threat_level_id] + 1 threat_levels_counts[threat_level_id] = threat_levels_counts[threat_level_id] + 1
threat_level_id = threat_levels[threat_level_id] threat_level_id = threat_levels[threat_level_id]
analysis_id = int(event_data['analysis']) analysis_id = int(event_data['analysis'])
analysis_completion_counts[analysis_id] = analysis_completion_counts[analysis_id] + 1 analysis_completion_counts[analysis_id] = analysis_completion_counts[analysis_id] + 1
analysis = analysis_completion[analysis_id] analysis = analysis_completion[analysis_id]
report['misp_events'].append({'id': event_data['id'], 'title': event_data['info'].replace('\n', '').encode('utf-8'), 'date': event_data['date'], 'timestamp': timestamp, 'publish_timestamp': publish_timestamp, 'threat_level': threat_level_id, 'analysis_completion': analysis}) report['misp_events'].append({'id': event_data['id'], 'title': event_data['info'].replace('\n', '').encode('utf-8'), 'date': event_data['date'], 'timestamp': timestamp, 'publish_timestamp': publish_timestamp, 'threat_level': threat_level_id, 'analysis_completion': analysis})
# Walk through the attributes # Walk through the attributes
if 'Attribute' in event_data: if 'Attribute' in event_data:
event_attr = event_data['Attribute'] event_attr = event_data['Attribute']
for attr in event_attr: for attr in event_attr:
number_of_attributes = number_of_attributes + 1 number_of_attributes = number_of_attributes + 1
type = attr['type'] type = attr['type']
category = attr['category'] category = attr['category']
to_ids = attr['to_ids'] to_ids = attr['to_ids']
if to_ids: if to_ids:
number_of_attributes_to_ids = number_of_attributes_to_ids + 1 number_of_attributes_to_ids = number_of_attributes_to_ids + 1
if type in attr_type: if type in attr_type:
attr_type[type] = attr_type[type] + 1 attr_type[type] = attr_type[type] + 1
else: else:
attr_type[type] = 1 attr_type[type] = 1
if category in attr_category: if category in attr_category:
attr_category[category] = attr_category[category] + 1 attr_category[category] = attr_category[category] + 1
else: else:
attr_category[category] = 1 attr_category[category] = 1
# Process tags # Process tags
if 'Tag' in event_data: if 'Tag' in event_data:
tags_attr = event_data['Tag'] tags_attr = event_data['Tag']
for tag in tags_attr: for tag in tags_attr:
tag_title = tag['name'] tag_title = tag['name']
if tag_title.lower().replace(' ', '') in tags_tlp: if tag_title.lower().replace(' ', '') in tags_tlp:
tags_tlp[tag_title.lower().replace(' ', '')] = tags_tlp[tag_title.lower().replace(' ', '')] + 1 tags_tlp[tag_title.lower().replace(' ', '')] = tags_tlp[tag_title.lower().replace(' ', '')] + 1
if 'misp-galaxy:mitre-' in tag_title: if 'misp-galaxy:mitre-' in tag_title:
if tag_title in tags_misp_galaxy_mitre: if tag_title in tags_misp_galaxy_mitre:
tags_misp_galaxy_mitre[tag_title] = tags_misp_galaxy_mitre[tag_title] + 1 tags_misp_galaxy_mitre[tag_title] = tags_misp_galaxy_mitre[tag_title] + 1
else: else:
tags_misp_galaxy_mitre[tag_title] = 1 tags_misp_galaxy_mitre[tag_title] = 1
if 'misp-galaxy:threat-actor=' in tag_title: if 'misp-galaxy:threat-actor=' in tag_title:
if tag_title in tags_misp_galaxy_threat_actor: if tag_title in tags_misp_galaxy_threat_actor:
tags_misp_galaxy_threat_actor[tag_title] = tags_misp_galaxy_threat_actor[tag_title] + 1 tags_misp_galaxy_threat_actor[tag_title] = tags_misp_galaxy_threat_actor[tag_title] + 1
else: else:
tags_misp_galaxy_threat_actor[tag_title] = 1 tags_misp_galaxy_threat_actor[tag_title] = 1
elif 'misp-galaxy:' in tag_title: elif 'misp-galaxy:' in tag_title:
if tag_title in tags_misp_galaxy: if tag_title in tags_misp_galaxy:
tags_misp_galaxy[tag_title] = tags_misp_galaxy[tag_title] + 1 tags_misp_galaxy[tag_title] = tags_misp_galaxy[tag_title] + 1
else: else:
tags_misp_galaxy[tag_title] = 1 tags_misp_galaxy[tag_title] = 1
if tag_title in tags_type: if tag_title in tags_type:
tags_type[tag_title] = tags_type[tag_title] + 1 tags_type[tag_title] = tags_type[tag_title] + 1
else: else:
tags_type[tag_title] = 1 tags_type[tag_title] = 1
# Process the galaxies # Process the galaxies
if 'Galaxy' in event_data: if 'Galaxy' in event_data:
galaxy_attr = event_data['Galaxy'] galaxy_attr = event_data['Galaxy']
for galaxy in galaxy_attr: for galaxy in galaxy_attr:
galaxy_title = galaxy['type'] galaxy_title = galaxy['type']
if galaxy_title in galaxies: if galaxy_title in galaxies:
galaxies[galaxy_title] = galaxies[galaxy_title] + 1 galaxies[galaxy_title] = galaxies[galaxy_title] + 1
else: else:
galaxies[galaxy_title] = 1 galaxies[galaxy_title] = 1
for cluster in galaxy['GalaxyCluster']: for cluster in galaxy['GalaxyCluster']:
cluster_value = cluster['type'] cluster_value = cluster['type']
if cluster_value in galaxies_cluster: if cluster_value in galaxies_cluster:
galaxies_cluster[cluster_value] = galaxies_cluster[cluster_value] + 1 galaxies_cluster[cluster_value] = galaxies_cluster[cluster_value] + 1
else: else:
galaxies_cluster[cluster_value] = 1 galaxies_cluster[cluster_value] = 1
report['number_of_attributes'] = number_of_attributes report['number_of_attributes'] = number_of_attributes
report['number_of_attributes_to_ids'] = number_of_attributes_to_ids report['number_of_attributes_to_ids'] = number_of_attributes_to_ids
report['attr_type'] = attr_type report['attr_type'] = attr_type
report['attr_category'] = attr_category report['attr_category'] = attr_category
report['tags_type'] = tags_type report['tags_type'] = tags_type
report['tags_tlp'] = tags_tlp report['tags_tlp'] = tags_tlp
report['tags_misp_galaxy_mitre'] = tags_misp_galaxy_mitre report['tags_misp_galaxy_mitre'] = tags_misp_galaxy_mitre
report['tags_misp_galaxy'] = tags_misp_galaxy report['tags_misp_galaxy'] = tags_misp_galaxy
report['tags_misp_galaxy_threat_actor'] = tags_misp_galaxy_threat_actor report['tags_misp_galaxy_threat_actor'] = tags_misp_galaxy_threat_actor
report['galaxies'] = galaxies report['galaxies'] = galaxies
report['galaxies_cluster'] = galaxies_cluster report['galaxies_cluster'] = galaxies_cluster
# General MISP statistics # General MISP statistics
user_statistics = misp.users_statistics() user_statistics = misp.users_statistics()
if user_statistics and 'errors' not in user_statistics: if user_statistics and 'errors' not in user_statistics:
report['user_statistics'] = user_statistics report['user_statistics'] = user_statistics
# Return the report data # Return the report data
return report return report
except Exception as e: except Exception as e:
sys.exit('Unable to get statistics from MISP') sys.exit('Unable to get statistics from MISP')
def build_report(report, timeframe, misp_url, sanitize_report=True): def build_report(report, timeframe, misp_url, sanitize_report=True):
''' '''
Build the body of the report and optional attachments Build the body of the report and optional attachments
''' '''
attachments = {} attachments = {}
now = datetime.now() now = datetime.now()
current_date = now.strftime(ts_format) current_date = now.strftime(ts_format)
if timeframe: if timeframe:
report_body = "MISP Report %s for last %s on %s\n-------------------------------------------------------------------------------" % (current_date, timeframe, misp_url) report_body = "MISP Report %s for last %s on %s\n-------------------------------------------------------------------------------" % (current_date, timeframe, misp_url)
else: else:
report_body = "MISP Report %s from %s to %s on %s\n-------------------------------------------------------------------------------" % (current_date, date_from, date_to, misp_url) report_body = "MISP Report %s from %s to %s on %s\n-------------------------------------------------------------------------------" % (current_date, date_from, date_to, misp_url)
report_body = report_body + '\nNew or updated events: %s' % report['number_of_misp_events'] report_body = report_body + '\nNew or updated events: %s' % report['number_of_misp_events']
report_body = report_body + '\nNew or updated attributes: %s' % report['number_of_attributes'] report_body = report_body + '\nNew or updated attributes: %s' % report['number_of_attributes']
report_body = report_body + '\nNew or updated attributes with IDS flag: %s' % report['number_of_attributes_to_ids'] report_body = report_body + '\nNew or updated attributes with IDS flag: %s' % report['number_of_attributes_to_ids']
report_body = report_body + '\n' report_body = report_body + '\n'
if 'user_statistics' in report: if 'user_statistics' in report:
report_body = report_body + '\nTotal events: %s' % report['user_statistics']['stats']['event_count'] report_body = report_body + '\nTotal events: %s' % report['user_statistics']['stats']['event_count']
report_body = report_body + '\nTotal attributes: %s' % report['user_statistics']['stats']['attribute_count'] report_body = report_body + '\nTotal attributes: %s' % report['user_statistics']['stats']['attribute_count']
report_body = report_body + '\nTotal users: %s' % report['user_statistics']['stats']['user_count'] report_body = report_body + '\nTotal users: %s' % report['user_statistics']['stats']['user_count']
report_body = report_body + '\nTotal orgs: %s' % report['user_statistics']['stats']['org_count'] report_body = report_body + '\nTotal orgs: %s' % report['user_statistics']['stats']['org_count']
report_body = report_body + '\nTotal correlation: %s' % report['user_statistics']['stats']['correlation_count'] report_body = report_body + '\nTotal correlation: %s' % report['user_statistics']['stats']['correlation_count']
report_body = report_body + '\nTotal proposals: %s' % report['user_statistics']['stats']['proposal_count'] report_body = report_body + '\nTotal proposals: %s' % report['user_statistics']['stats']['proposal_count']
report_body = report_body + '\n\n' report_body = report_body + '\n\n'
if args.mispevent: if args.mispevent:
report_body = report_body + '\nNew or updated events\n-------------------------------------------------------------------------------' report_body = report_body + '\nNew or updated events\n-------------------------------------------------------------------------------'
attachments['misp_events'] = 'ID;Title;Date;Updated;Published;ThreatLevel;AnalysisStatus' attachments['misp_events'] = 'ID;Title;Date;Updated;Published;ThreatLevel;AnalysisStatus'
for el in report['misp_events']: for el in report['misp_events']:
report_body = report_body + '\n #%s %s (%s) \t%s \n\t\t\t\t(Date: %s, Updated: %s, Published: %s)' % (el['id'], el['threat_level'], el['analysis_completion'], el['title'].decode('utf-8'), el['date'], el['timestamp'], el['publish_timestamp']) report_body = report_body + '\n #%s %s (%s) \t%s \n\t\t\t\t(Date: %s, Updated: %s, Published: %s)' % (el['id'], el['threat_level'], el['analysis_completion'], el['title'].decode('utf-8'), el['date'], el['timestamp'], el['publish_timestamp'])
attachments['misp_events'] = attachments['misp_events'] + '\n%s;%s;%s;%s;%s;%s;%s' % (el['id'], el['title'].decode('utf-8'), el['date'], el['timestamp'], el['publish_timestamp'], el['threat_level'], el['analysis_completion']) attachments['misp_events'] = attachments['misp_events'] + '\n%s;%s;%s;%s;%s;%s;%s' % (el['id'], el['title'].decode('utf-8'), el['date'], el['timestamp'], el['publish_timestamp'], el['threat_level'], el['analysis_completion'])
report_body, attachments['attr_category'] = add_report_body(report_body, 'New or updated attributes - Category', report['attr_category'], 'AttributeCategory;Qt') report_body, attachments['attr_category'] = add_report_body(report_body, 'New or updated attributes - Category', report['attr_category'], 'AttributeCategory;Qt')
report_body, attachments['attr_type'] = add_report_body(report_body, 'New or updated attributes - Type', report['attr_type'], 'AttributeType;Qt') report_body, attachments['attr_type'] = add_report_body(report_body, 'New or updated attributes - Type', report['attr_type'], 'AttributeType;Qt')
report_body, attachments['tags_tlp'] = add_report_body(report_body, 'TLP Codes', report['tags_tlp'], 'TLP;Qt') report_body, attachments['tags_tlp'] = add_report_body(report_body, 'TLP Codes', report['tags_tlp'], 'TLP;Qt')
report_body, attachments['tags_misp_galaxy'] = add_report_body(report_body, 'Tag MISP Galaxy', report['tags_misp_galaxy'], 'MISPGalaxy;Qt') report_body, attachments['tags_misp_galaxy'] = add_report_body(report_body, 'Tag MISP Galaxy', report['tags_misp_galaxy'], 'MISPGalaxy;Qt')
report_body, attachments['tags_misp_galaxy_mitre'] = add_report_body(report_body, 'Tag MISP Galaxy Mitre', report['tags_misp_galaxy_mitre'], 'MISPGalaxyMitre;Qt') report_body, attachments['tags_misp_galaxy_mitre'] = add_report_body(report_body, 'Tag MISP Galaxy Mitre', report['tags_misp_galaxy_mitre'], 'MISPGalaxyMitre;Qt')
report_body, attachments['tags_misp_galaxy_threat_actor'] = add_report_body(report_body, 'Tag MISP Galaxy Threat Actor', report['tags_misp_galaxy_threat_actor'], 'MISPGalaxyThreatActor;Qt') report_body, attachments['tags_misp_galaxy_threat_actor'] = add_report_body(report_body, 'Tag MISP Galaxy Threat Actor', report['tags_misp_galaxy_threat_actor'], 'MISPGalaxyThreatActor;Qt')
report_body, attachments['tags_type'] = add_report_body(report_body, 'Tags', report['tags_type'], 'Tag;Qt') report_body, attachments['tags_type'] = add_report_body(report_body, 'Tags', report['tags_type'], 'Tag;Qt')
report_body, attachments['galaxies'] = add_report_body(report_body, 'Galaxies', report['galaxies'], 'Galaxies;Qt') report_body, attachments['galaxies'] = add_report_body(report_body, 'Galaxies', report['galaxies'], 'Galaxies;Qt')
report_body, attachments['galaxies_cluster'] = add_report_body(report_body, 'Galaxies Cluster', report['galaxies_cluster'], 'Galaxies;Qt') report_body, attachments['galaxies_cluster'] = add_report_body(report_body, 'Galaxies Cluster', report['galaxies_cluster'], 'Galaxies;Qt')
if sanitize_report: if sanitize_report:
mitre_tactic = get_sanitized_report(report['tags_misp_galaxy_mitre'], 'ATT&CK Tactic') mitre_tactic = get_sanitized_report(report['tags_misp_galaxy_mitre'], 'ATT&CK Tactic')
mitre_group = get_sanitized_report(report['tags_misp_galaxy_mitre'], 'ATT&CK Group') mitre_group = get_sanitized_report(report['tags_misp_galaxy_mitre'], 'ATT&CK Group')
mitre_software = get_sanitized_report(report['tags_misp_galaxy_mitre'], 'ATT&CK Software') mitre_software = get_sanitized_report(report['tags_misp_galaxy_mitre'], 'ATT&CK Software')
threat_actor = get_sanitized_report(report['tags_misp_galaxy_threat_actor'], 'MISP Threat Actor') threat_actor = get_sanitized_report(report['tags_misp_galaxy_threat_actor'], 'MISP Threat Actor')
misp_tag = get_sanitized_report(report['tags_type'], 'MISP Tags', False, True) misp_tag = get_sanitized_report(report['tags_type'], 'MISP Tags', False, True)
report_body, attachments['mitre_tactics'] = add_report_body(report_body, 'MITRE ATT&CK Tactics (sanitized)', mitre_tactic, 'MITRETactics;Qt') report_body, attachments['mitre_tactics'] = add_report_body(report_body, 'MITRE ATT&CK Tactics (sanitized)', mitre_tactic, 'MITRETactics;Qt')
report_body, attachments['mitre_group'] = add_report_body(report_body, 'MITRE ATT&CK Group (sanitized)', mitre_group, 'MITREGroup;Qt') report_body, attachments['mitre_group'] = add_report_body(report_body, 'MITRE ATT&CK Group (sanitized)', mitre_group, 'MITREGroup;Qt')
report_body, attachments['mitre_software'] = add_report_body(report_body, 'MITRE ATT&CK Software (sanitized)', mitre_software, 'MITRESoftware;Qt') report_body, attachments['mitre_software'] = add_report_body(report_body, 'MITRE ATT&CK Software (sanitized)', mitre_software, 'MITRESoftware;Qt')
report_body, attachments['threat_actor'] = add_report_body(report_body, 'MISP Threat Actor (sanitized)', threat_actor, 'MISPThreatActor;Qt') report_body, attachments['threat_actor'] = add_report_body(report_body, 'MISP Threat Actor (sanitized)', threat_actor, 'MISPThreatActor;Qt')
report_body, attachments['misp_tag'] = add_report_body(report_body, 'Tags (sanitized)', misp_tag, 'MISPTags;Qt') report_body, attachments['misp_tag'] = add_report_body(report_body, 'Tags (sanitized)', misp_tag, 'MISPTags;Qt')
report_body = report_body + "\n\nMISP Reporter Finished\n" report_body = report_body + "\n\nMISP Reporter Finished\n"
return report_body, attachments return report_body, attachments
def add_report_body(report_body, subtitle, data_object, csv_title): def add_report_body(report_body, subtitle, data_object, csv_title):
''' '''
Add a section to the report body text Add a section to the report body text
''' '''
if report_body: if report_body:
report_body = report_body + '\n\n' report_body = report_body + '\n\n'
report_body = report_body + '\n%s\n-------------------------------------------------------------------------------' % subtitle report_body = report_body + '\n%s\n-------------------------------------------------------------------------------' % subtitle
data_object_s = sorted(data_object.items(), key=lambda kv: (kv[1], kv[0]), reverse=True) data_object_s = sorted(data_object.items(), key=lambda kv: (kv[1], kv[0]), reverse=True)
csv_attachment = csv_title csv_attachment = csv_title
for el in data_object_s: for el in data_object_s:
report_body = report_body + "\n%s \t %s" % (el[0], el[1]) report_body = report_body + "\n%s \t %s" % (el[0], el[1])
csv_attachment = csv_attachment + '\n%s;%s' % (el[0], el[1]) csv_attachment = csv_attachment + '\n%s;%s' % (el[0], el[1])
return report_body, csv_attachment return report_body, csv_attachment
def msg_attach(content, filename): def msg_attach(content, filename):
''' '''
Return an message attachment object Return an message attachment object
''' '''
part = MIMEBase('application', "octet-stream") part = MIMEBase('application', "octet-stream")
part.set_payload(content) part.set_payload(content)
part.add_header('Content-Disposition', 'attachment; filename="%s"' % filename) part.add_header('Content-Disposition', 'attachment; filename="%s"' % filename)
return part return part
def print_report(report_body, attachments, smtp_from, smtp_to, smtp_server, misp_url): def print_report(report_body, attachments, smtp_from, smtp_to, smtp_server, misp_url):
''' '''
Print (or send) the report Print (or send) the report
''' '''
if args.mail: if args.mail:
now = datetime.now() now = datetime.now()
current_date = now.strftime(ts_format) current_date = now.strftime(ts_format)
if timeframe: if timeframe:
subject = "MISP Report %s for last %s on %s" % (current_date, timeframe, misp_url) subject = "MISP Report %s for last %s on %s" % (current_date, timeframe, misp_url)
else: else:
subject = "MISP Report %s from %s to %s on %s" % (current_date, date_from, date_to, misp_url) subject = "MISP Report %s from %s to %s on %s" % (current_date, date_from, date_to, misp_url)
msg = MIMEMultipart() msg = MIMEMultipart()
msg['From'] = smtp_from msg['From'] = smtp_from
msg['To'] = smtp_to msg['To'] = smtp_to
msg['Subject'] = subject msg['Subject'] = subject
msg.attach(MIMEText(report_body, 'text')) msg.attach(MIMEText(report_body, 'text'))
if args.mispevent: if args.mispevent:
part = MIMEBase('application', "octet-stream") part = MIMEBase('application', "octet-stream")
part.set_payload(attachments['misp_events']) part.set_payload(attachments['misp_events'])
part.add_header('Content-Disposition', 'attachment; filename="misp_events.csv"') part.add_header('Content-Disposition', 'attachment; filename="misp_events.csv"')
msg.attach(part) msg.attach(part)
msg.attach(msg_attach(attachments['attr_type'], 'attr_type.csv')) msg.attach(msg_attach(attachments['attr_type'], 'attr_type.csv'))
msg.attach(msg_attach(attachments['attr_category'], 'attr_category.csv')) msg.attach(msg_attach(attachments['attr_category'], 'attr_category.csv'))
msg.attach(msg_attach(attachments['tags_tlp'], 'tags_tlp.csv')) msg.attach(msg_attach(attachments['tags_tlp'], 'tags_tlp.csv'))
msg.attach(msg_attach(attachments['tags_misp_galaxy_mitre'], 'tags_misp_galaxy_mitre.csv')) msg.attach(msg_attach(attachments['tags_misp_galaxy_mitre'], 'tags_misp_galaxy_mitre.csv'))
msg.attach(msg_attach(attachments['tags_misp_galaxy'], 'tags_misp_galaxy.csv')) msg.attach(msg_attach(attachments['tags_misp_galaxy'], 'tags_misp_galaxy.csv'))
msg.attach(msg_attach(attachments['tags_misp_galaxy_threat_actor'], 'tags_misp_galaxy_threat_actor.csv')) msg.attach(msg_attach(attachments['tags_misp_galaxy_threat_actor'], 'tags_misp_galaxy_threat_actor.csv'))
msg.attach(msg_attach(attachments['tags_type'], 'tags_type.csv')) msg.attach(msg_attach(attachments['tags_type'], 'tags_type.csv'))
msg.attach(msg_attach(attachments['galaxies'], 'galaxies.csv')) msg.attach(msg_attach(attachments['galaxies'], 'galaxies.csv'))
msg.attach(msg_attach(attachments['galaxies_cluster'], 'galaxies_cluster.csv')) msg.attach(msg_attach(attachments['galaxies_cluster'], 'galaxies_cluster.csv'))
msg.attach(msg_attach(attachments['misp_tag'], 'misp_tag.csv')) msg.attach(msg_attach(attachments['misp_tag'], 'misp_tag.csv'))
msg.attach(msg_attach(attachments['threat_actor'], 'threat_actor.csv')) msg.attach(msg_attach(attachments['threat_actor'], 'threat_actor.csv'))
msg.attach(msg_attach(attachments['mitre_software'], 'mitre_software.csv')) msg.attach(msg_attach(attachments['mitre_software'], 'mitre_software.csv'))
msg.attach(msg_attach(attachments['mitre_group'], 'mitre_group.csv')) msg.attach(msg_attach(attachments['mitre_group'], 'mitre_group.csv'))
msg.attach(msg_attach(attachments['mitre_tactics'], 'mitre_tactics.csv')) msg.attach(msg_attach(attachments['mitre_tactics'], 'mitre_tactics.csv'))
server = smtplib.SMTP(smtp_server) server = smtplib.SMTP(smtp_server)
server.sendmail(smtp_from, smtp_to, msg.as_string()) server.sendmail(smtp_from, smtp_to, msg.as_string())
else: else:
print(report_body) print(report_body)
def get_sanitized_report(dataset, sanitize_selector='ATT&CK Tactic', lower=False, add_not_sanitized=False): def get_sanitized_report(dataset, sanitize_selector='ATT&CK Tactic', lower=False, add_not_sanitized=False):
''' '''
Remove or bundle some of the tags Remove or bundle some of the tags
'quick'n'dirty ; could also do this by using the galaxy/tags definition 'quick'n'dirty ; could also do this by using the galaxy/tags definition
''' '''
# If you add the element completely then it gets removed by an empty string; this allows to filter out non-relevant items # If you add the element completely then it gets removed by an empty string; this allows to filter out non-relevant items
sanitize_set = { sanitize_set = {
'ATT&CK Tactic': ['misp-galaxy:mitre-enterprise-attack-pattern="', 'misp-galaxy:mitre-pre-attack-pattern="', 'misp-galaxy:mitre-mobile-attack-pattern="', 'misp-galaxy:mitre-attack-pattern="', 'misp-galaxy:mitre-enterprise-attack-attack-pattern="', 'misp-galaxy:mitre-pre-attack-attack-pattern="', 'misp-galaxy:mitre-enterprise-attack-attack-pattern="', 'misp-galaxy:mitre-mobile-attack-attack-pattern="'], 'ATT&CK Tactic': ['misp-galaxy:mitre-enterprise-attack-pattern="', 'misp-galaxy:mitre-pre-attack-pattern="', 'misp-galaxy:mitre-mobile-attack-pattern="', 'misp-galaxy:mitre-attack-pattern="', 'misp-galaxy:mitre-enterprise-attack-attack-pattern="', 'misp-galaxy:mitre-pre-attack-attack-pattern="', 'misp-galaxy:mitre-enterprise-attack-attack-pattern="', 'misp-galaxy:mitre-mobile-attack-attack-pattern="'],
'ATT&CK Group': ['misp-galaxy:mitre-enterprise-intrusion-set="', 'misp-galaxy:mitre-pre-intrusion-set="', 'misp-galaxy:mitre-mobile-intrusion-set="', 'misp-galaxy:mitre-intrusion-set="', 'misp-galaxy:mitre-enterprise-attack-intrusion-set="', 'misp-galaxy:mitre-pre-attack-intrusion-set="', 'misp-galaxy:mitre-mobile-attack-intrusion-set="'], 'ATT&CK Group': ['misp-galaxy:mitre-enterprise-intrusion-set="', 'misp-galaxy:mitre-pre-intrusion-set="', 'misp-galaxy:mitre-mobile-intrusion-set="', 'misp-galaxy:mitre-intrusion-set="', 'misp-galaxy:mitre-enterprise-attack-intrusion-set="', 'misp-galaxy:mitre-pre-attack-intrusion-set="', 'misp-galaxy:mitre-mobile-attack-intrusion-set="'],
'ATT&CK Software': ['misp-galaxy:mitre-enterprise-malware="', 'misp-galaxy:mitre-pre-malware="', 'misp-galaxy:mitre-mobile-malware="', 'misp-galaxy:mitre-malware="', 'misp-galaxy:mitre-enterprise-attack-tool="', 'misp-galaxy:mitre-enterprise-tool="', 'misp-galaxy:mitre-pre-tool="', 'misp-galaxy:mitre-mobile-tool="', 'misp-galaxy:mitre-tool="', 'misp-galaxy:mitre-enterprise-attack-malware="'], 'ATT&CK Software': ['misp-galaxy:mitre-enterprise-malware="', 'misp-galaxy:mitre-pre-malware="', 'misp-galaxy:mitre-mobile-malware="', 'misp-galaxy:mitre-malware="', 'misp-galaxy:mitre-enterprise-attack-tool="', 'misp-galaxy:mitre-enterprise-tool="', 'misp-galaxy:mitre-pre-tool="', 'misp-galaxy:mitre-mobile-tool="', 'misp-galaxy:mitre-tool="', 'misp-galaxy:mitre-enterprise-attack-malware="'],
'MISP Threat Actor': ['misp-galaxy:threat-actor="'], 'MISP Threat Actor': ['misp-galaxy:threat-actor="'],
'MISP Tags': ['circl:incident-classification="', 'osint:source-type="blog-post"', 'misp-galaxy:tool="', 'CERT-XLM:malicious-code="', 'circl:topic="', 'ddos:type="', 'ecsirt:fraud="', 'dnc:malware-type="', 'enisa:nefarious-activity-abuse="', 'europol-incident:information-gathering="', 'misp-galaxy:ransomware="', 'misp-galaxy:rat="', 'misp-galaxy:social-dark-patterns="', 'misp-galaxy:tool="', 'misp:threat-level="', 'ms-caro-malware:malware-platform=', 'ms-caro-malware:malware-type=', 'veris:security_incident="', 'veris:attribute:integrity:variety="', 'veris:actor:motive="', 'misp-galaxy:banker="', 'misp-galaxy:malpedia="', 'misp-galaxy:botnet="', 'malware_classification:malware-category="', 'TLP: white', 'TLP: Green', 'MISP Tags': ['circl:incident-classification="', 'osint:source-type="blog-post"', 'misp-galaxy:tool="', 'CERT-XLM:malicious-code="', 'circl:topic="', 'ddos:type="', 'ecsirt:fraud="', 'dnc:malware-type="', 'enisa:nefarious-activity-abuse="', 'europol-incident:information-gathering="', 'misp-galaxy:ransomware="', 'misp-galaxy:rat="', 'misp-galaxy:social-dark-patterns="', 'misp-galaxy:tool="', 'misp:threat-level="', 'ms-caro-malware:malware-platform=', 'ms-caro-malware:malware-type=', 'veris:security_incident="', 'veris:attribute:integrity:variety="', 'veris:actor:motive="', 'misp-galaxy:banker="', 'misp-galaxy:malpedia="', 'misp-galaxy:botnet="', 'malware_classification:malware-category="', 'TLP: white', 'TLP: Green',
'inthreat:event-src="feed-osint"', 'tlp:white', 'tlp:amber', 'tlp:green', 'tlp:red', 'osint:source-type="blog-post"', 'Partner Feed', 'IBM XForce', 'type:OSINT', 'malware:', 'osint:lifetime="perpetual"', 'Actor:', 'osint:certainty="50"', 'Banker:', 'Group:', 'Threat:', 'inthreat:event-src="feed-osint"', 'tlp:white', 'tlp:amber', 'tlp:green', 'tlp:red', 'osint:source-type="blog-post"', 'Partner Feed', 'IBM XForce', 'type:OSINT', 'malware:', 'osint:lifetime="perpetual"', 'Actor:', 'osint:certainty="50"', 'Banker:', 'Group:', 'Threat:',
'ncsc-nl-ndn:feed="selected"', 'misp-galaxy:microsoft-activity-group="', 'admiralty-scale:source-reliability="b"', 'admiralty-scale:source-reliability="a"', 'admiralty-scale:information-credibility="2"', 'admiralty-scale:information-credibility="3"', 'ncsc-nl-ndn:feed="selected"', 'misp-galaxy:microsoft-activity-group="', 'admiralty-scale:source-reliability="b"', 'admiralty-scale:source-reliability="a"', 'admiralty-scale:information-credibility="2"', 'admiralty-scale:information-credibility="3"',
'feed:source="CESICAT"', 'osint:source-type="automatic-analysis"', 'workflow:state="complete"', 'osint:source-type="technical-report"', 'feed:source="CESICAT"', 'osint:source-type="automatic-analysis"', 'workflow:state="complete"', 'osint:source-type="technical-report"',
'csirt_case_classification:incident-category="', 'dnc:driveby-type="', 'veris:action:social:variety="', 'osint:source-type="', 'csirt_case_classification:incident-category="', 'dnc:driveby-type="', 'veris:action:social:variety="', 'osint:source-type="',
'osint:source-type="microblog-post"', 'ecsirt:malicious-code="', 'misp-galaxy:sector="', 'veris:action:variety=', 'label=', 'csirt_case_classification:incident-category="', 'admiralty-scale:source-reliability="c"', 'workflow:todo="review"', 'LDO-CERT:detection="toSIEM"', 'Threat tlp:White', 'Threat Type:', 'adversary:infrastructure-state="active"', 'cirl:incident-classification:', 'misp-galaxy:android="', 'dnc:infrastructure-type="', 'ecsirt:information-gathering="', 'ecsirt:intrusions="', 'dhs-ciip-sectors:DHS-critical-sectors="', 'malware_classification:obfuscation-technique="no-obfuscation"', 'osint:source-type="microblog-post"', 'ecsirt:malicious-code="', 'misp-galaxy:sector="', 'veris:action:variety=', 'label=', 'csirt_case_classification:incident-category="', 'admiralty-scale:source-reliability="c"', 'workflow:todo="review"', 'LDO-CERT:detection="toSIEM"', 'Threat tlp:White', 'Threat Type:', 'adversary:infrastructure-state="active"', 'cirl:incident-classification:', 'misp-galaxy:android="', 'dnc:infrastructure-type="', 'ecsirt:information-gathering="', 'ecsirt:intrusions="', 'dhs-ciip-sectors:DHS-critical-sectors="', 'malware_classification:obfuscation-technique="no-obfuscation"',
'riskiq:threat-type="', 'veris:action:hacking:variety="', 'veris:action:social:target="', 'workflow:state="incomplete"', 'workflow:todo="add-tagging"', 'workflow:todo="add-context"', 'europol-incident:availability="', 'label=', 'misp-galaxy:stealer="', 'misp-galaxy:exploit-kit="', 'rsit:availability="', 'rsit:fraud="', 'ransomware:type="', 'veris:action:variety=', 'malware:', 'riskiq:threat-type="', 'veris:action:hacking:variety="', 'veris:action:social:target="', 'workflow:state="incomplete"', 'workflow:todo="add-tagging"', 'workflow:todo="add-context"', 'europol-incident:availability="', 'label=', 'misp-galaxy:stealer="', 'misp-galaxy:exploit-kit="', 'rsit:availability="', 'rsit:fraud="', 'ransomware:type="', 'veris:action:variety=', 'malware:',
'ecsirt:abusive-content="']} 'ecsirt:abusive-content="']}
if sanitize_selector == 'MISP Tags': if sanitize_selector == 'MISP Tags':
sanitize_set['MISP Tags'] = sanitize_set['MISP Tags'] + sanitize_set['ATT&CK Tactic'] + sanitize_set['ATT&CK Group'] + sanitize_set['ATT&CK Software'] + sanitize_set['MISP Threat Actor'] sanitize_set['MISP Tags'] = sanitize_set['MISP Tags'] + sanitize_set['ATT&CK Tactic'] + sanitize_set['ATT&CK Group'] + sanitize_set['ATT&CK Software'] + sanitize_set['MISP Threat Actor']
result_sanitize_set = {} result_sanitize_set = {}
if dataset: if dataset:
for element in dataset: for element in dataset:
sanited = False sanited = False
for sanitize_el in sanitize_set[sanitize_selector]: for sanitize_el in sanitize_set[sanitize_selector]:
if sanitize_el in element: if sanitize_el in element:
sanited = True sanited = True
new_el = element.replace(sanitize_el, '').replace('"', '').strip() new_el = element.replace(sanitize_el, '').replace('"', '').strip()
if lower: if lower:
new_el = new_el.lower() new_el = new_el.lower()
result_sanitize_set[new_el] = dataset[element] result_sanitize_set[new_el] = dataset[element]
if add_not_sanitized and not sanited: if add_not_sanitized and not sanited:
new_el = element.strip() new_el = element.strip()
if lower: if lower:
new_el = new_el.lower() new_el = new_el.lower()
result_sanitize_set[new_el] = dataset[element] result_sanitize_set[new_el] = dataset[element]
return result_sanitize_set return result_sanitize_set
if __name__ == '__main__': if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Generate a report of your MISP statistics.') parser = argparse.ArgumentParser(description='Generate a report of your MISP statistics.')
group = parser.add_mutually_exclusive_group(required=True) group = parser.add_mutually_exclusive_group(required=True)
group.add_argument('-t', '--timeframe', action='store', help='Timeframe to include in the report') group.add_argument('-t', '--timeframe', action='store', help='Timeframe to include in the report')
group.add_argument('-f', '--date_from', action='store', help='Start date of query (YYYY-MM-DD)') group.add_argument('-f', '--date_from', action='store', help='Start date of query (YYYY-MM-DD)')
parser.add_argument('-u', '---date-to', action='store', help='End date of query (YYYY-MM-DD)') parser.add_argument('-u', '---date-to', action='store', help='End date of query (YYYY-MM-DD)')
parser.add_argument('-e', '--mispevent', action='store_true', help='Include MISP event titles') parser.add_argument('-e', '--mispevent', action='store_true', help='Include MISP event titles')
parser.add_argument('-m', '--mail', action='store_true', help='Mail the report') parser.add_argument('-m', '--mail', action='store_true', help='Mail the report')
parser.add_argument('-o', '--mailoptions', action='store', help='mailoptions: \'smtp_from=INSERT_FROM;smtp_to=INSERT_TO;smtp_server=localhost\'') parser.add_argument('-o', '--mailoptions', action='store', help='mailoptions: \'smtp_from=INSERT_FROM;smtp_to=INSERT_TO;smtp_server=localhost\'')
args = parser.parse_args() args = parser.parse_args()
misp = init(misp_url, misp_key, misp_verifycert) misp = init(misp_url, misp_key, misp_verifycert)
timeframe = args.timeframe timeframe = args.timeframe
if not timeframe: if not timeframe:
date_from = args.date_from date_from = args.date_from
if not args.date_to: if not args.date_to:
today = date.today() today = date.today()
date_to = today.strftime("%Y-%m-%d") date_to = today.strftime("%Y-%m-%d")
else: else:
date_to = args.date_to date_to = args.date_to
else: else:
date_from = None date_from = None
date_to = None date_to = None
ts_format = '%Y-%m-%d %H:%M:%S' ts_format = '%Y-%m-%d %H:%M:%S'
threat_levels = ['High', 'Medium', 'Low', 'Undef'] threat_levels = ['High', 'Medium', 'Low', 'Undef']
analysis_completion = ['Initial', 'Ongoing', 'Complete'] analysis_completion = ['Initial', 'Ongoing', 'Complete']
smtp_from = 'INSERT_FROM' smtp_from = 'INSERT_FROM'
smtp_to = 'INSERT_TO' smtp_to = 'INSERT_TO'
smtp_server = 'localhost' smtp_server = 'localhost'
if args.mailoptions: if args.mailoptions:
mailoptions = args.mailoptions.split(';') mailoptions = args.mailoptions.split(';')
for s in mailoptions: for s in mailoptions:
if s.split('=')[0] == 'smtp_from': if s.split('=')[0] == 'smtp_from':
smtp_from = s.split('=')[1] smtp_from = s.split('=')[1]
if s.split('=')[0] == 'smtp_to': if s.split('=')[0] == 'smtp_to':
smtp_to = s.split('=')[1] smtp_to = s.split('=')[1]
if s.split('=')[0] == 'smtp_server': if s.split('=')[0] == 'smtp_server':
smtp_server = s.split('=')[1] smtp_server = s.split('=')[1]
report = get_data(misp, timeframe, date_from, date_to) report = get_data(misp, timeframe, date_from, date_to)
if(report): if(report):
report_body, attachments = build_report(report, timeframe, misp_url) report_body, attachments = build_report(report, timeframe, misp_url)
print_report(report_body, attachments, smtp_from, smtp_to, smtp_server, misp_url) print_report(report_body, attachments, smtp_from, smtp_to, smtp_server, misp_url)