mirror of https://github.com/MISP/PyMISP
Merge branch 'master' into python3.6
Raphaël Vinot
commit
6d4ae575e9
|
|
@ -1,405 +1,405 @@
|
||||||
#!/usr/bin/env python
|
#!/usr/bin/env python
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
'''
|
'''
|
||||||
Koen Van Impe
|
Koen Van Impe
|
||||||
Maxime Thiebaut
|
Maxime Thiebaut
|
||||||
|
|
||||||
Generate a report of your MISP statistics
|
Generate a report of your MISP statistics
|
||||||
Put this script in crontab to run every /15 or /60
|
Put this script in crontab to run every /15 or /60
|
||||||
*/5 * * * * mispuser /usr/bin/python3 /home/mispuser/PyMISP/examples/stats_report.py -t 30d -m -v
|
*/5 * * * * mispuser /usr/bin/python3 /home/mispuser/PyMISP/examples/stats_report.py -t 30d -m -v
|
||||||
|
|
||||||
Do inline config in "main"
|
Do inline config in "main"
|
||||||
|
|
||||||
'''
|
'''
|
||||||
|
|
||||||
from pymisp import ExpandedPyMISP
|
from pymisp import ExpandedPyMISP
|
||||||
from keys import misp_url, misp_key, misp_verifycert
|
from keys import misp_url, misp_key, misp_verifycert
|
||||||
import argparse
|
import argparse
|
||||||
import os
|
import os
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from datetime import date
|
from datetime import date
|
||||||
import time
|
import time
|
||||||
import sys
|
import sys
|
||||||
import smtplib
|
import smtplib
|
||||||
import mimetypes
|
import mimetypes
|
||||||
from email.mime.multipart import MIMEMultipart
|
from email.mime.multipart import MIMEMultipart
|
||||||
from email import encoders
|
from email import encoders
|
||||||
from email.mime.base import MIMEBase
|
from email.mime.base import MIMEBase
|
||||||
from email.mime.text import MIMEText
|
from email.mime.text import MIMEText
|
||||||
|
|
||||||
# Suppress those "Unverified HTTPS request is being made"
|
# Suppress those "Unverified HTTPS request is being made"
|
||||||
import urllib3
|
import urllib3
|
||||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||||
|
|
||||||
|
|
||||||
|
def init(url, key, verifycert):
|
||||||
def init(url, key, verifycert):
|
'''
|
||||||
'''
|
Template to get MISP module started
|
||||||
Template to get MISP module started
|
'''
|
||||||
'''
|
return ExpandedPyMISP(url, key, verifycert, 'json')
|
||||||
return ExpandedPyMISP(url, key, verifycert, 'json')
|
|
||||||
|
|
||||||
|
def get_data(misp, timeframe, date_from=None, date_to=None):
|
||||||
|
'''
|
||||||
def get_data(misp, timeframe, date_from = None, date_to = None):
|
Get the event date to build our report
|
||||||
'''
|
'''
|
||||||
Get the event date to build our report
|
number_of_misp_events = 0
|
||||||
'''
|
number_of_attributes = 0
|
||||||
number_of_misp_events = 0
|
number_of_attributes_to_ids = 0
|
||||||
number_of_attributes = 0
|
attr_type = {}
|
||||||
number_of_attributes_to_ids = 0
|
attr_category = {}
|
||||||
attr_type = {}
|
tags_type = {}
|
||||||
attr_category = {}
|
tags_tlp = {'tlp:white': 0, 'tlp:green': 0, 'tlp:amber': 0, 'tlp:red': 0}
|
||||||
tags_type = {}
|
tags_misp_galaxy_mitre = {}
|
||||||
tags_tlp = {'tlp:white': 0, 'tlp:green': 0, 'tlp:amber': 0, 'tlp:red': 0}
|
tags_misp_galaxy = {}
|
||||||
tags_misp_galaxy_mitre = {}
|
tags_misp_galaxy_threat_actor = {}
|
||||||
tags_misp_galaxy = {}
|
galaxies = {}
|
||||||
tags_misp_galaxy_threat_actor = {}
|
galaxies_cluster = {}
|
||||||
galaxies = {}
|
threat_levels_counts = [0, 0, 0, 0]
|
||||||
galaxies_cluster = {}
|
analysis_completion_counts = [0, 0, 0]
|
||||||
threat_levels_counts = [0, 0, 0, 0]
|
report = {}
|
||||||
analysis_completion_counts = [0, 0, 0]
|
|
||||||
report = {}
|
try:
|
||||||
|
if date_from and date_to:
|
||||||
try:
|
stats_event_response = misp.search(date_from=date_from, date_to=date_to)
|
||||||
if date_from and date_to:
|
else:
|
||||||
stats_event_response = misp.search(date_from=date_from, date_to=date_to)
|
stats_event_response = misp.search(last=timeframe)
|
||||||
else:
|
|
||||||
stats_event_response = misp.search(last=timeframe)
|
# Number of new or updated events since timestamp
|
||||||
|
report['number_of_misp_events'] = len(stats_event_response)
|
||||||
# Number of new or updated events since timestamp
|
report['misp_events'] = []
|
||||||
report['number_of_misp_events'] = len(stats_event_response)
|
|
||||||
report['misp_events'] = []
|
for event in stats_event_response:
|
||||||
|
event_data = event['Event']
|
||||||
for event in stats_event_response:
|
|
||||||
event_data = event['Event']
|
timestamp = datetime.utcfromtimestamp(int(event_data['timestamp'])).strftime(ts_format)
|
||||||
|
publish_timestamp = datetime.utcfromtimestamp(int(event_data['publish_timestamp'])).strftime(ts_format)
|
||||||
timestamp = datetime.utcfromtimestamp(int(event_data['timestamp'])).strftime(ts_format)
|
|
||||||
publish_timestamp = datetime.utcfromtimestamp(int(event_data['publish_timestamp'])).strftime(ts_format)
|
threat_level_id = int(event_data['threat_level_id']) - 1
|
||||||
|
threat_levels_counts[threat_level_id] = threat_levels_counts[threat_level_id] + 1
|
||||||
threat_level_id = int(event_data['threat_level_id']) - 1
|
threat_level_id = threat_levels[threat_level_id]
|
||||||
threat_levels_counts[threat_level_id] = threat_levels_counts[threat_level_id] + 1
|
|
||||||
threat_level_id = threat_levels[threat_level_id]
|
analysis_id = int(event_data['analysis'])
|
||||||
|
analysis_completion_counts[analysis_id] = analysis_completion_counts[analysis_id] + 1
|
||||||
analysis_id = int(event_data['analysis'])
|
analysis = analysis_completion[analysis_id]
|
||||||
analysis_completion_counts[analysis_id] = analysis_completion_counts[analysis_id] + 1
|
|
||||||
analysis = analysis_completion[analysis_id]
|
report['misp_events'].append({'id': event_data['id'], 'title': event_data['info'].replace('\n', '').encode('utf-8'), 'date': event_data['date'], 'timestamp': timestamp, 'publish_timestamp': publish_timestamp, 'threat_level': threat_level_id, 'analysis_completion': analysis})
|
||||||
|
|
||||||
report['misp_events'].append({'id': event_data['id'], 'title': event_data['info'].replace('\n', '').encode('utf-8'), 'date': event_data['date'], 'timestamp': timestamp, 'publish_timestamp': publish_timestamp, 'threat_level': threat_level_id, 'analysis_completion': analysis})
|
# Walk through the attributes
|
||||||
|
if 'Attribute' in event_data:
|
||||||
# Walk through the attributes
|
event_attr = event_data['Attribute']
|
||||||
if 'Attribute' in event_data:
|
for attr in event_attr:
|
||||||
event_attr = event_data['Attribute']
|
number_of_attributes = number_of_attributes + 1
|
||||||
for attr in event_attr:
|
|
||||||
number_of_attributes = number_of_attributes + 1
|
type = attr['type']
|
||||||
|
category = attr['category']
|
||||||
type = attr['type']
|
to_ids = attr['to_ids']
|
||||||
category = attr['category']
|
|
||||||
to_ids = attr['to_ids']
|
if to_ids:
|
||||||
|
number_of_attributes_to_ids = number_of_attributes_to_ids + 1
|
||||||
if to_ids:
|
|
||||||
number_of_attributes_to_ids = number_of_attributes_to_ids + 1
|
if type in attr_type:
|
||||||
|
attr_type[type] = attr_type[type] + 1
|
||||||
if type in attr_type:
|
else:
|
||||||
attr_type[type] = attr_type[type] + 1
|
attr_type[type] = 1
|
||||||
else:
|
|
||||||
attr_type[type] = 1
|
if category in attr_category:
|
||||||
|
attr_category[category] = attr_category[category] + 1
|
||||||
if category in attr_category:
|
else:
|
||||||
attr_category[category] = attr_category[category] + 1
|
attr_category[category] = 1
|
||||||
else:
|
|
||||||
attr_category[category] = 1
|
# Process tags
|
||||||
|
if 'Tag' in event_data:
|
||||||
# Process tags
|
tags_attr = event_data['Tag']
|
||||||
if 'Tag' in event_data:
|
for tag in tags_attr:
|
||||||
tags_attr = event_data['Tag']
|
tag_title = tag['name']
|
||||||
for tag in tags_attr:
|
|
||||||
tag_title = tag['name']
|
if tag_title.lower().replace(' ', '') in tags_tlp:
|
||||||
|
tags_tlp[tag_title.lower().replace(' ', '')] = tags_tlp[tag_title.lower().replace(' ', '')] + 1
|
||||||
if tag_title.lower().replace(' ', '') in tags_tlp:
|
|
||||||
tags_tlp[tag_title.lower().replace(' ', '')] = tags_tlp[tag_title.lower().replace(' ', '')] + 1
|
if 'misp-galaxy:mitre-' in tag_title:
|
||||||
|
if tag_title in tags_misp_galaxy_mitre:
|
||||||
if 'misp-galaxy:mitre-' in tag_title:
|
tags_misp_galaxy_mitre[tag_title] = tags_misp_galaxy_mitre[tag_title] + 1
|
||||||
if tag_title in tags_misp_galaxy_mitre:
|
else:
|
||||||
tags_misp_galaxy_mitre[tag_title] = tags_misp_galaxy_mitre[tag_title] + 1
|
tags_misp_galaxy_mitre[tag_title] = 1
|
||||||
else:
|
|
||||||
tags_misp_galaxy_mitre[tag_title] = 1
|
if 'misp-galaxy:threat-actor=' in tag_title:
|
||||||
|
if tag_title in tags_misp_galaxy_threat_actor:
|
||||||
if 'misp-galaxy:threat-actor=' in tag_title:
|
tags_misp_galaxy_threat_actor[tag_title] = tags_misp_galaxy_threat_actor[tag_title] + 1
|
||||||
if tag_title in tags_misp_galaxy_threat_actor:
|
else:
|
||||||
tags_misp_galaxy_threat_actor[tag_title] = tags_misp_galaxy_threat_actor[tag_title] + 1
|
tags_misp_galaxy_threat_actor[tag_title] = 1
|
||||||
else:
|
elif 'misp-galaxy:' in tag_title:
|
||||||
tags_misp_galaxy_threat_actor[tag_title] = 1
|
if tag_title in tags_misp_galaxy:
|
||||||
elif 'misp-galaxy:' in tag_title:
|
tags_misp_galaxy[tag_title] = tags_misp_galaxy[tag_title] + 1
|
||||||
if tag_title in tags_misp_galaxy:
|
else:
|
||||||
tags_misp_galaxy[tag_title] = tags_misp_galaxy[tag_title] + 1
|
tags_misp_galaxy[tag_title] = 1
|
||||||
else:
|
|
||||||
tags_misp_galaxy[tag_title] = 1
|
if tag_title in tags_type:
|
||||||
|
tags_type[tag_title] = tags_type[tag_title] + 1
|
||||||
if tag_title in tags_type:
|
else:
|
||||||
tags_type[tag_title] = tags_type[tag_title] + 1
|
tags_type[tag_title] = 1
|
||||||
else:
|
|
||||||
tags_type[tag_title] = 1
|
# Process the galaxies
|
||||||
|
if 'Galaxy' in event_data:
|
||||||
# Process the galaxies
|
galaxy_attr = event_data['Galaxy']
|
||||||
if 'Galaxy' in event_data:
|
for galaxy in galaxy_attr:
|
||||||
galaxy_attr = event_data['Galaxy']
|
galaxy_title = galaxy['type']
|
||||||
for galaxy in galaxy_attr:
|
|
||||||
galaxy_title = galaxy['type']
|
if galaxy_title in galaxies:
|
||||||
|
galaxies[galaxy_title] = galaxies[galaxy_title] + 1
|
||||||
if galaxy_title in galaxies:
|
else:
|
||||||
galaxies[galaxy_title] = galaxies[galaxy_title] + 1
|
galaxies[galaxy_title] = 1
|
||||||
else:
|
|
||||||
galaxies[galaxy_title] = 1
|
for cluster in galaxy['GalaxyCluster']:
|
||||||
|
cluster_value = cluster['type']
|
||||||
for cluster in galaxy['GalaxyCluster']:
|
if cluster_value in galaxies_cluster:
|
||||||
cluster_value = cluster['type']
|
galaxies_cluster[cluster_value] = galaxies_cluster[cluster_value] + 1
|
||||||
if cluster_value in galaxies_cluster:
|
else:
|
||||||
galaxies_cluster[cluster_value] = galaxies_cluster[cluster_value] + 1
|
galaxies_cluster[cluster_value] = 1
|
||||||
else:
|
report['number_of_attributes'] = number_of_attributes
|
||||||
galaxies_cluster[cluster_value] = 1
|
report['number_of_attributes_to_ids'] = number_of_attributes_to_ids
|
||||||
report['number_of_attributes'] = number_of_attributes
|
report['attr_type'] = attr_type
|
||||||
report['number_of_attributes_to_ids'] = number_of_attributes_to_ids
|
report['attr_category'] = attr_category
|
||||||
report['attr_type'] = attr_type
|
report['tags_type'] = tags_type
|
||||||
report['attr_category'] = attr_category
|
report['tags_tlp'] = tags_tlp
|
||||||
report['tags_type'] = tags_type
|
report['tags_misp_galaxy_mitre'] = tags_misp_galaxy_mitre
|
||||||
report['tags_tlp'] = tags_tlp
|
report['tags_misp_galaxy'] = tags_misp_galaxy
|
||||||
report['tags_misp_galaxy_mitre'] = tags_misp_galaxy_mitre
|
report['tags_misp_galaxy_threat_actor'] = tags_misp_galaxy_threat_actor
|
||||||
report['tags_misp_galaxy'] = tags_misp_galaxy
|
report['galaxies'] = galaxies
|
||||||
report['tags_misp_galaxy_threat_actor'] = tags_misp_galaxy_threat_actor
|
report['galaxies_cluster'] = galaxies_cluster
|
||||||
report['galaxies'] = galaxies
|
|
||||||
report['galaxies_cluster'] = galaxies_cluster
|
# General MISP statistics
|
||||||
|
user_statistics = misp.users_statistics()
|
||||||
# General MISP statistics
|
if user_statistics and 'errors' not in user_statistics:
|
||||||
user_statistics = misp.users_statistics()
|
report['user_statistics'] = user_statistics
|
||||||
if user_statistics and 'errors' not in user_statistics:
|
|
||||||
report['user_statistics'] = user_statistics
|
# Return the report data
|
||||||
|
return report
|
||||||
# Return the report data
|
except Exception as e:
|
||||||
return report
|
sys.exit('Unable to get statistics from MISP')
|
||||||
except Exception as e:
|
|
||||||
sys.exit('Unable to get statistics from MISP')
|
|
||||||
|
def build_report(report, timeframe, misp_url, sanitize_report=True):
|
||||||
|
'''
|
||||||
|
Build the body of the report and optional attachments
|
||||||
def build_report(report, timeframe, misp_url):
|
'''
|
||||||
'''
|
attachments = {}
|
||||||
Build the body of the report and optional attachments
|
|
||||||
'''
|
now = datetime.now()
|
||||||
attachments = {}
|
current_date = now.strftime(ts_format)
|
||||||
|
if timeframe:
|
||||||
now = datetime.now()
|
report_body = "MISP Report %s for last %s on %s\n-------------------------------------------------------------------------------" % (current_date, timeframe, misp_url)
|
||||||
current_date = now.strftime(ts_format)
|
else:
|
||||||
if timeframe:
|
report_body = "MISP Report %s from %s to %s on %s\n-------------------------------------------------------------------------------" % (current_date, date_from, date_to, misp_url)
|
||||||
report_body = "MISP Report %s for last %s on %s\n-------------------------------------------------------------------------------" % (current_date, timeframe, misp_url)
|
|
||||||
else:
|
report_body = report_body + '\nNew or updated events: %s' % report['number_of_misp_events']
|
||||||
report_body = "MISP Report %s from %s to %s on %s\n-------------------------------------------------------------------------------" % (current_date, date_from, date_to, misp_url)
|
report_body = report_body + '\nNew or updated attributes: %s' % report['number_of_attributes']
|
||||||
|
report_body = report_body + '\nNew or updated attributes with IDS flag: %s' % report['number_of_attributes_to_ids']
|
||||||
report_body = report_body + '\nNew or updated events: %s' % report['number_of_misp_events']
|
report_body = report_body + '\n'
|
||||||
report_body = report_body + '\nNew or updated attributes: %s' % report['number_of_attributes']
|
if 'user_statistics' in report:
|
||||||
report_body = report_body + '\nNew or updated attributes with IDS flag: %s' % report['number_of_attributes_to_ids']
|
report_body = report_body + '\nTotal events: %s' % report['user_statistics']['stats']['event_count']
|
||||||
report_body = report_body + '\n'
|
report_body = report_body + '\nTotal attributes: %s' % report['user_statistics']['stats']['attribute_count']
|
||||||
if 'user_statistics' in report:
|
report_body = report_body + '\nTotal users: %s' % report['user_statistics']['stats']['user_count']
|
||||||
report_body = report_body + '\nTotal events: %s' % report['user_statistics']['stats']['event_count']
|
report_body = report_body + '\nTotal orgs: %s' % report['user_statistics']['stats']['org_count']
|
||||||
report_body = report_body + '\nTotal attributes: %s' % report['user_statistics']['stats']['attribute_count']
|
report_body = report_body + '\nTotal correlation: %s' % report['user_statistics']['stats']['correlation_count']
|
||||||
report_body = report_body + '\nTotal users: %s' % report['user_statistics']['stats']['user_count']
|
report_body = report_body + '\nTotal proposals: %s' % report['user_statistics']['stats']['proposal_count']
|
||||||
report_body = report_body + '\nTotal orgs: %s' % report['user_statistics']['stats']['org_count']
|
|
||||||
report_body = report_body + '\nTotal correlation: %s' % report['user_statistics']['stats']['correlation_count']
|
report_body = report_body + '\n\n'
|
||||||
report_body = report_body + '\nTotal proposals: %s' % report['user_statistics']['stats']['proposal_count']
|
|
||||||
|
if args.mispevent:
|
||||||
report_body = report_body + '\n\n'
|
report_body = report_body + '\nNew or updated events\n-------------------------------------------------------------------------------'
|
||||||
|
attachments['misp_events'] = 'ID;Title;Date;Updated;Published;ThreatLevel;AnalysisStatus'
|
||||||
if args.mispevent:
|
for el in report['misp_events']:
|
||||||
report_body = report_body + '\nNew or updated events\n-------------------------------------------------------------------------------'
|
report_body = report_body + '\n #%s %s (%s) \t%s \n\t\t\t\t(Date: %s, Updated: %s, Published: %s)' % (el['id'], el['threat_level'], el['analysis_completion'], el['title'].decode('utf-8'), el['date'], el['timestamp'], el['publish_timestamp'])
|
||||||
attachments['misp_events'] = 'ID;Title;Date;Updated;Published;ThreatLevel;AnalysisStatus'
|
attachments['misp_events'] = attachments['misp_events'] + '\n%s;%s;%s;%s;%s;%s;%s' % (el['id'], el['title'].decode('utf-8'), el['date'], el['timestamp'], el['publish_timestamp'], el['threat_level'], el['analysis_completion'])
|
||||||
for el in report['misp_events']:
|
|
||||||
report_body = report_body + '\n #%s %s (%s) \t%s \n\t\t\t\t(Date: %s, Updated: %s, Published: %s)' % (el['id'], el['threat_level'], el['analysis_completion'], el['title'].decode('utf-8'), el['date'], el['timestamp'], el['publish_timestamp'])
|
report_body, attachments['attr_category'] = add_report_body(report_body, 'New or updated attributes - Category', report['attr_category'], 'AttributeCategory;Qt')
|
||||||
attachments['misp_events'] = attachments['misp_events'] + '\n%s;%s;%s;%s;%s;%s;%s' % (el['id'], el['title'].decode('utf-8'), el['date'], el['timestamp'], el['publish_timestamp'], el['threat_level'], el['analysis_completion'])
|
report_body, attachments['attr_type'] = add_report_body(report_body, 'New or updated attributes - Type', report['attr_type'], 'AttributeType;Qt')
|
||||||
|
report_body, attachments['tags_tlp'] = add_report_body(report_body, 'TLP Codes', report['tags_tlp'], 'TLP;Qt')
|
||||||
report_body = report_body + '\n\n'
|
report_body, attachments['tags_misp_galaxy'] = add_report_body(report_body, 'Tag MISP Galaxy', report['tags_misp_galaxy'], 'MISPGalaxy;Qt')
|
||||||
|
report_body, attachments['tags_misp_galaxy_mitre'] = add_report_body(report_body, 'Tag MISP Galaxy Mitre', report['tags_misp_galaxy_mitre'], 'MISPGalaxyMitre;Qt')
|
||||||
report_body = report_body + '\nNew or updated attributes - Category \n-------------------------------------------------------------------------------'
|
report_body, attachments['tags_misp_galaxy_threat_actor'] = add_report_body(report_body, 'Tag MISP Galaxy Threat Actor', report['tags_misp_galaxy_threat_actor'], 'MISPGalaxyThreatActor;Qt')
|
||||||
attr_category_s = sorted(report['attr_category'].items(), key=lambda kv: (kv[1], kv[0]), reverse=True)
|
report_body, attachments['tags_type'] = add_report_body(report_body, 'Tags', report['tags_type'], 'Tag;Qt')
|
||||||
attachments['attr_category'] = 'AttributeCategory;Qt'
|
report_body, attachments['galaxies'] = add_report_body(report_body, 'Galaxies', report['galaxies'], 'Galaxies;Qt')
|
||||||
for el in attr_category_s:
|
report_body, attachments['galaxies_cluster'] = add_report_body(report_body, 'Galaxies Cluster', report['galaxies_cluster'], 'Galaxies;Qt')
|
||||||
report_body = report_body + '\n%s \t %s' % (el[0], el[1])
|
|
||||||
attachments['attr_category'] = attachments['attr_category'] + '\n%s;%s' % (el[0], el[1])
|
if sanitize_report:
|
||||||
|
mitre_tactic = get_sanitized_report(report['tags_misp_galaxy_mitre'], 'ATT&CK Tactic')
|
||||||
report_body = report_body + '\n\n'
|
mitre_group = get_sanitized_report(report['tags_misp_galaxy_mitre'], 'ATT&CK Group')
|
||||||
|
mitre_software = get_sanitized_report(report['tags_misp_galaxy_mitre'], 'ATT&CK Software')
|
||||||
report_body = report_body + '\nNew or updated attributes - Type \n-------------------------------------------------------------------------------'
|
threat_actor = get_sanitized_report(report['tags_misp_galaxy_threat_actor'], 'MISP Threat Actor')
|
||||||
attr_type_s = sorted(report['attr_type'].items(), key=lambda kv: (kv[1], kv[0]), reverse=True)
|
misp_tag = get_sanitized_report(report['tags_type'], 'MISP Tags', False, True)
|
||||||
attachments['attr_type'] = 'AttributeType;Qt'
|
|
||||||
for el in attr_type_s:
|
report_body, attachments['mitre_tactics'] = add_report_body(report_body, 'MITRE ATT&CK Tactics (sanitized)', mitre_tactic, 'MITRETactics;Qt')
|
||||||
report_body = report_body + '\n%s \t %s' % (el[0], el[1])
|
report_body, attachments['mitre_group'] = add_report_body(report_body, 'MITRE ATT&CK Group (sanitized)', mitre_group, 'MITREGroup;Qt')
|
||||||
attachments['attr_type'] = attachments['attr_type'] + '\n%s;%s' % (el[0], el[1])
|
report_body, attachments['mitre_software'] = add_report_body(report_body, 'MITRE ATT&CK Software (sanitized)', mitre_software, 'MITRESoftware;Qt')
|
||||||
|
report_body, attachments['threat_actor'] = add_report_body(report_body, 'MISP Threat Actor (sanitized)', threat_actor, 'MISPThreatActor;Qt')
|
||||||
report_body = report_body + '\n\n'
|
report_body, attachments['misp_tag'] = add_report_body(report_body, 'Tags (sanitized)', misp_tag, 'MISPTags;Qt')
|
||||||
|
|
||||||
report_body = report_body + '\nTLP Codes \n-------------------------------------------------------------------------------'
|
report_body = report_body + "\n\nMISP Reporter Finished\n"
|
||||||
attachments['tags_tlp'] = 'TLP;Qt'
|
|
||||||
for el in report['tags_tlp']:
|
return report_body, attachments
|
||||||
report_body = report_body + "\n%s \t %s" % (el, report['tags_tlp'][el])
|
|
||||||
attachments['tags_tlp'] = attachments['tags_tlp'] + '\n%s;%s' % (el, report['tags_tlp'][el])
|
|
||||||
|
def add_report_body(report_body, subtitle, data_object, csv_title):
|
||||||
report_body = report_body + '\n\n'
|
'''
|
||||||
|
Add a section to the report body text
|
||||||
report_body = report_body + '\nTag MISP Galaxy\n-------------------------------------------------------------------------------'
|
'''
|
||||||
tags_misp_galaxy_s = sorted(report['tags_misp_galaxy'].items(), key=lambda kv: (kv[1], kv[0]), reverse=True)
|
if report_body:
|
||||||
attachments['tags_misp_galaxy'] = 'MISPGalaxy;Qt'
|
report_body = report_body + '\n\n'
|
||||||
for el in tags_misp_galaxy_s:
|
report_body = report_body + '\n%s\n-------------------------------------------------------------------------------' % subtitle
|
||||||
report_body = report_body + "\n%s \t %s" % (el[0], el[1])
|
data_object_s = sorted(data_object.items(), key=lambda kv: (kv[1], kv[0]), reverse=True)
|
||||||
attachments['tags_misp_galaxy'] = attachments['tags_misp_galaxy'] + '\n%s;%s' % (el[0], el[1])
|
csv_attachment = csv_title
|
||||||
|
for el in data_object_s:
|
||||||
report_body = report_body + '\n\n'
|
report_body = report_body + "\n%s \t %s" % (el[0], el[1])
|
||||||
|
csv_attachment = csv_attachment + '\n%s;%s' % (el[0], el[1])
|
||||||
report_body = report_body + '\nTag MISP Galaxy Mitre \n-------------------------------------------------------------------------------'
|
|
||||||
tags_misp_galaxy_mitre_s = sorted(report['tags_misp_galaxy_mitre'].items(), key=lambda kv: (kv[1], kv[0]), reverse=True)
|
return report_body, csv_attachment
|
||||||
attachments['tags_misp_galaxy_mitre'] = 'MISPGalaxyMitre;Qt'
|
|
||||||
for el in tags_misp_galaxy_mitre_s:
|
|
||||||
report_body = report_body + "\n%s \t %s" % (el[0], el[1])
|
def msg_attach(content, filename):
|
||||||
attachments['tags_misp_galaxy_mitre'] = attachments['tags_misp_galaxy_mitre'] + '\n%s;%s' % (el[0], el[1])
|
'''
|
||||||
|
Return an message attachment object
|
||||||
report_body = report_body + '\n\n'
|
'''
|
||||||
|
part = MIMEBase('application', "octet-stream")
|
||||||
report_body = report_body + '\nTag MISP Galaxy Threat Actor \n-------------------------------------------------------------------------------'
|
part.set_payload(content)
|
||||||
tags_misp_galaxy_threat_actor_s = sorted(report['tags_misp_galaxy_threat_actor'].items(), key=lambda kv: (kv[1], kv[0]), reverse=True)
|
part.add_header('Content-Disposition', 'attachment; filename="%s"' % filename)
|
||||||
attachments['tags_misp_galaxy_threat_actor'] = 'MISPGalaxyThreatActor;Qt'
|
return part
|
||||||
for el in tags_misp_galaxy_threat_actor_s:
|
|
||||||
report_body = report_body + "\n%s \t %s" % (el[0], el[1])
|
|
||||||
attachments['tags_misp_galaxy_threat_actor'] = attachments['tags_misp_galaxy_threat_actor'] + '\n%s;%s' % (el[0], el[1])
|
def print_report(report_body, attachments, smtp_from, smtp_to, smtp_server, misp_url):
|
||||||
|
'''
|
||||||
report_body = report_body + '\n\n'
|
Print (or send) the report
|
||||||
|
'''
|
||||||
report_body = report_body + '\nTags \n-------------------------------------------------------------------------------'
|
if args.mail:
|
||||||
tags_type_s = sorted(report['tags_type'].items(), key=lambda kv: (kv[1], kv[0]), reverse=True)
|
now = datetime.now()
|
||||||
attachments['tags_type'] = 'Tag;Qt'
|
current_date = now.strftime(ts_format)
|
||||||
for el in tags_type_s:
|
|
||||||
report_body = report_body + "\n%s \t %s" % (el[0], el[1])
|
if timeframe:
|
||||||
attachments['tags_type'] = attachments['tags_type'] + '\n%s;%s' % (el[0], el[1])
|
subject = "MISP Report %s for last %s on %s" % (current_date, timeframe, misp_url)
|
||||||
|
else:
|
||||||
report_body = report_body + '\n\n'
|
subject = "MISP Report %s from %s to %s on %s" % (current_date, date_from, date_to, misp_url)
|
||||||
|
|
||||||
report_body = report_body + '\nGalaxies \n-------------------------------------------------------------------------------'
|
msg = MIMEMultipart()
|
||||||
galaxies_s = sorted(report['galaxies'].items(), key=lambda kv: (kv[1], kv[0]), reverse=True)
|
msg['From'] = smtp_from
|
||||||
attachments['galaxies'] = 'Galaxies;Qt'
|
msg['To'] = smtp_to
|
||||||
for el in galaxies_s:
|
msg['Subject'] = subject
|
||||||
report_body = report_body + "\n%s \t %s" % (el[0], el[1])
|
|
||||||
attachments['galaxies'] = attachments['galaxies'] + '\n%s;%s' % (el[0], el[1])
|
msg.attach(MIMEText(report_body, 'text'))
|
||||||
|
|
||||||
report_body = report_body + '\n\n'
|
if args.mispevent:
|
||||||
|
part = MIMEBase('application', "octet-stream")
|
||||||
report_body = report_body + '\nGalaxies Cluster \n-------------------------------------------------------------------------------'
|
part.set_payload(attachments['misp_events'])
|
||||||
galaxies_cluster_s = sorted(report['galaxies_cluster'].items(), key=lambda kv: (kv[1], kv[0]), reverse=True)
|
part.add_header('Content-Disposition', 'attachment; filename="misp_events.csv"')
|
||||||
attachments['galaxies_cluster'] = 'Galaxies;Qt'
|
msg.attach(part)
|
||||||
for el in galaxies_cluster_s:
|
|
||||||
report_body = report_body + "\n%s \t %s" % (el[0], el[1])
|
msg.attach(msg_attach(attachments['attr_type'], 'attr_type.csv'))
|
||||||
attachments['galaxies_cluster'] = attachments['galaxies_cluster'] + '\n%s;%s' % (el[0], el[1])
|
msg.attach(msg_attach(attachments['attr_category'], 'attr_category.csv'))
|
||||||
|
msg.attach(msg_attach(attachments['tags_tlp'], 'tags_tlp.csv'))
|
||||||
report_body = report_body + "\n\nMISP Reporter Finished\n"
|
msg.attach(msg_attach(attachments['tags_misp_galaxy_mitre'], 'tags_misp_galaxy_mitre.csv'))
|
||||||
|
msg.attach(msg_attach(attachments['tags_misp_galaxy'], 'tags_misp_galaxy.csv'))
|
||||||
return report_body, attachments
|
msg.attach(msg_attach(attachments['tags_misp_galaxy_threat_actor'], 'tags_misp_galaxy_threat_actor.csv'))
|
||||||
|
msg.attach(msg_attach(attachments['tags_type'], 'tags_type.csv'))
|
||||||
|
msg.attach(msg_attach(attachments['galaxies'], 'galaxies.csv'))
|
||||||
|
msg.attach(msg_attach(attachments['galaxies_cluster'], 'galaxies_cluster.csv'))
|
||||||
def msg_attach(content, filename):
|
msg.attach(msg_attach(attachments['misp_tag'], 'misp_tag.csv'))
|
||||||
'''
|
msg.attach(msg_attach(attachments['threat_actor'], 'threat_actor.csv'))
|
||||||
Return an message attachment object
|
msg.attach(msg_attach(attachments['mitre_software'], 'mitre_software.csv'))
|
||||||
'''
|
msg.attach(msg_attach(attachments['mitre_group'], 'mitre_group.csv'))
|
||||||
part = MIMEBase('application', "octet-stream")
|
msg.attach(msg_attach(attachments['mitre_tactics'], 'mitre_tactics.csv'))
|
||||||
part.set_payload(content)
|
|
||||||
part.add_header('Content-Disposition', 'attachment; filename="%s"' % filename)
|
server = smtplib.SMTP(smtp_server)
|
||||||
return part
|
server.sendmail(smtp_from, smtp_to, msg.as_string())
|
||||||
|
|
||||||
|
else:
|
||||||
|
print(report_body)
|
||||||
def print_report(report_body, attachments, smtp_from, smtp_to, smtp_server, misp_url):
|
|
||||||
'''
|
|
||||||
Print (or send) the report
|
def get_sanitized_report(dataset, sanitize_selector='ATT&CK Tactic', lower=False, add_not_sanitized=False):
|
||||||
'''
|
'''
|
||||||
if args.mail:
|
Remove or bundle some of the tags
|
||||||
now = datetime.now()
|
'quick'n'dirty ; could also do this by using the galaxy/tags definition
|
||||||
current_date = now.strftime(ts_format)
|
'''
|
||||||
|
# If you add the element completely then it gets removed by an empty string; this allows to filter out non-relevant items
|
||||||
if timeframe:
|
sanitize_set = {
|
||||||
subject = "MISP Report %s for last %s on %s" % (current_date, timeframe, misp_url)
|
'ATT&CK Tactic': ['misp-galaxy:mitre-enterprise-attack-pattern="', 'misp-galaxy:mitre-pre-attack-pattern="', 'misp-galaxy:mitre-mobile-attack-pattern="', 'misp-galaxy:mitre-attack-pattern="', 'misp-galaxy:mitre-enterprise-attack-attack-pattern="', 'misp-galaxy:mitre-pre-attack-attack-pattern="', 'misp-galaxy:mitre-enterprise-attack-attack-pattern="', 'misp-galaxy:mitre-mobile-attack-attack-pattern="'],
|
||||||
else:
|
'ATT&CK Group': ['misp-galaxy:mitre-enterprise-intrusion-set="', 'misp-galaxy:mitre-pre-intrusion-set="', 'misp-galaxy:mitre-mobile-intrusion-set="', 'misp-galaxy:mitre-intrusion-set="', 'misp-galaxy:mitre-enterprise-attack-intrusion-set="', 'misp-galaxy:mitre-pre-attack-intrusion-set="', 'misp-galaxy:mitre-mobile-attack-intrusion-set="'],
|
||||||
subject = "MISP Report %s from %s to %s on %s" % (current_date, date_from, date_to, misp_url)
|
'ATT&CK Software': ['misp-galaxy:mitre-enterprise-malware="', 'misp-galaxy:mitre-pre-malware="', 'misp-galaxy:mitre-mobile-malware="', 'misp-galaxy:mitre-malware="', 'misp-galaxy:mitre-enterprise-attack-tool="', 'misp-galaxy:mitre-enterprise-tool="', 'misp-galaxy:mitre-pre-tool="', 'misp-galaxy:mitre-mobile-tool="', 'misp-galaxy:mitre-tool="', 'misp-galaxy:mitre-enterprise-attack-malware="'],
|
||||||
|
'MISP Threat Actor': ['misp-galaxy:threat-actor="'],
|
||||||
msg = MIMEMultipart()
|
'MISP Tags': ['circl:incident-classification="', 'osint:source-type="blog-post"', 'misp-galaxy:tool="', 'CERT-XLM:malicious-code="', 'circl:topic="', 'ddos:type="', 'ecsirt:fraud="', 'dnc:malware-type="', 'enisa:nefarious-activity-abuse="', 'europol-incident:information-gathering="', 'misp-galaxy:ransomware="', 'misp-galaxy:rat="', 'misp-galaxy:social-dark-patterns="', 'misp-galaxy:tool="', 'misp:threat-level="', 'ms-caro-malware:malware-platform=', 'ms-caro-malware:malware-type=', 'veris:security_incident="', 'veris:attribute:integrity:variety="', 'veris:actor:motive="', 'misp-galaxy:banker="', 'misp-galaxy:malpedia="', 'misp-galaxy:botnet="', 'malware_classification:malware-category="', 'TLP: white', 'TLP: Green',
|
||||||
msg['From'] = smtp_from
|
'inthreat:event-src="feed-osint"', 'tlp:white', 'tlp:amber', 'tlp:green', 'tlp:red', 'osint:source-type="blog-post"', 'Partner Feed', 'IBM XForce', 'type:OSINT', 'malware:', 'osint:lifetime="perpetual"', 'Actor:', 'osint:certainty="50"', 'Banker:', 'Group:', 'Threat:',
|
||||||
msg['To'] = smtp_to
|
'ncsc-nl-ndn:feed="selected"', 'misp-galaxy:microsoft-activity-group="', 'admiralty-scale:source-reliability="b"', 'admiralty-scale:source-reliability="a"', 'admiralty-scale:information-credibility="2"', 'admiralty-scale:information-credibility="3"',
|
||||||
msg['Subject'] = subject
|
'feed:source="CESICAT"', 'osint:source-type="automatic-analysis"', 'workflow:state="complete"', 'osint:source-type="technical-report"',
|
||||||
|
'csirt_case_classification:incident-category="', 'dnc:driveby-type="', 'veris:action:social:variety="', 'osint:source-type="',
|
||||||
msg.attach(MIMEText(report_body, 'text'))
|
'osint:source-type="microblog-post"', 'ecsirt:malicious-code="', 'misp-galaxy:sector="', 'veris:action:variety=', 'label=', 'csirt_case_classification:incident-category="', 'admiralty-scale:source-reliability="c"', 'workflow:todo="review"', 'LDO-CERT:detection="toSIEM"', 'Threat tlp:White', 'Threat Type:', 'adversary:infrastructure-state="active"', 'cirl:incident-classification:', 'misp-galaxy:android="', 'dnc:infrastructure-type="', 'ecsirt:information-gathering="', 'ecsirt:intrusions="', 'dhs-ciip-sectors:DHS-critical-sectors="', 'malware_classification:obfuscation-technique="no-obfuscation"',
|
||||||
|
'riskiq:threat-type="', 'veris:action:hacking:variety="', 'veris:action:social:target="', 'workflow:state="incomplete"', 'workflow:todo="add-tagging"', 'workflow:todo="add-context"', 'europol-incident:availability="', 'label=', 'misp-galaxy:stealer="', 'misp-galaxy:exploit-kit="', 'rsit:availability="', 'rsit:fraud="', 'ransomware:type="', 'veris:action:variety=', 'malware:',
|
||||||
if args.mispevent:
|
'ecsirt:abusive-content="']}
|
||||||
part = MIMEBase('application', "octet-stream")
|
if sanitize_selector == 'MISP Tags':
|
||||||
part.set_payload(attachments['misp_events'])
|
sanitize_set['MISP Tags'] = sanitize_set['MISP Tags'] + sanitize_set['ATT&CK Tactic'] + sanitize_set['ATT&CK Group'] + sanitize_set['ATT&CK Software'] + sanitize_set['MISP Threat Actor']
|
||||||
part.add_header('Content-Disposition', 'attachment; filename="misp_events.csv"')
|
result_sanitize_set = {}
|
||||||
msg.attach(part)
|
|
||||||
|
if dataset:
|
||||||
msg.attach(msg_attach(attachments['attr_type'], 'attr_type.csv'))
|
for element in dataset:
|
||||||
msg.attach(msg_attach(attachments['attr_category'], 'attr_category.csv'))
|
sanited = False
|
||||||
msg.attach(msg_attach(attachments['tags_tlp'], 'tags_tlp.csv'))
|
for sanitize_el in sanitize_set[sanitize_selector]:
|
||||||
msg.attach(msg_attach(attachments['tags_misp_galaxy_mitre'], 'tags_misp_galaxy_mitre.csv'))
|
if sanitize_el in element:
|
||||||
msg.attach(msg_attach(attachments['tags_misp_galaxy'], 'tags_misp_galaxy.csv'))
|
sanited = True
|
||||||
msg.attach(msg_attach(attachments['tags_misp_galaxy_threat_actor'], 'tags_misp_galaxy_threat_actor.csv'))
|
new_el = element.replace(sanitize_el, '').replace('"', '').strip()
|
||||||
msg.attach(msg_attach(attachments['tags_type'], 'tags_type.csv'))
|
if lower:
|
||||||
msg.attach(msg_attach(attachments['galaxies'], 'galaxies.csv'))
|
new_el = new_el.lower()
|
||||||
msg.attach(msg_attach(attachments['galaxies_cluster'], 'galaxies_cluster.csv'))
|
result_sanitize_set[new_el] = dataset[element]
|
||||||
|
if add_not_sanitized and not sanited:
|
||||||
server = smtplib.SMTP(smtp_server)
|
new_el = element.strip()
|
||||||
server.sendmail(smtp_from, smtp_to, msg.as_string())
|
if lower:
|
||||||
|
new_el = new_el.lower()
|
||||||
else:
|
result_sanitize_set[new_el] = dataset[element]
|
||||||
print(report_body)
|
|
||||||
|
return result_sanitize_set
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
parser = argparse.ArgumentParser(description='Generate a report of your MISP statistics.')
|
parser = argparse.ArgumentParser(description='Generate a report of your MISP statistics.')
|
||||||
group = parser.add_mutually_exclusive_group(required=True)
|
group = parser.add_mutually_exclusive_group(required=True)
|
||||||
group.add_argument('-t', '--timeframe',action='store', help='Timeframe to include in the report')
|
group.add_argument('-t', '--timeframe', action='store', help='Timeframe to include in the report')
|
||||||
group.add_argument('-f', '--date_from',action='store', help='Start date of query (YYYY-MM-DD)')
|
group.add_argument('-f', '--date_from', action='store', help='Start date of query (YYYY-MM-DD)')
|
||||||
parser.add_argument('-u', '---date-to', action='store', help='End date of query (YYYY-MM-DD)')
|
parser.add_argument('-u', '---date-to', action='store', help='End date of query (YYYY-MM-DD)')
|
||||||
parser.add_argument('-e', '--mispevent', action='store_true', help='Include MISP event titles')
|
parser.add_argument('-e', '--mispevent', action='store_true', help='Include MISP event titles')
|
||||||
parser.add_argument('-m', '--mail', action='store_true', help='Mail the report')
|
parser.add_argument('-m', '--mail', action='store_true', help='Mail the report')
|
||||||
parser.add_argument('-o', '--mailoptions', action='store', help='mailoptions: \'smtp_from=INSERT_FROM;smtp_to=INSERT_TO;smtp_server=localhost\'')
|
parser.add_argument('-o', '--mailoptions', action='store', help='mailoptions: \'smtp_from=INSERT_FROM;smtp_to=INSERT_TO;smtp_server=localhost\'')
|
||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
misp = init(misp_url, misp_key, misp_verifycert)
|
misp = init(misp_url, misp_key, misp_verifycert)
|
||||||
|
|
||||||
timeframe = args.timeframe
|
timeframe = args.timeframe
|
||||||
if not timeframe:
|
if not timeframe:
|
||||||
date_from = args.date_from
|
date_from = args.date_from
|
||||||
if not args.date_to:
|
if not args.date_to:
|
||||||
today = date.today()
|
today = date.today()
|
||||||
date_to = today.strftime("%Y-%m-%d")
|
date_to = today.strftime("%Y-%m-%d")
|
||||||
else:
|
else:
|
||||||
date_to = args.date_to
|
date_to = args.date_to
|
||||||
else:
|
else:
|
||||||
date_from = None
|
date_from = None
|
||||||
date_to = None
|
date_to = None
|
||||||
|
|
||||||
ts_format = '%Y-%m-%d %H:%M:%S'
|
ts_format = '%Y-%m-%d %H:%M:%S'
|
||||||
threat_levels = ['High', 'Medium', 'Low', 'Undef']
|
threat_levels = ['High', 'Medium', 'Low', 'Undef']
|
||||||
analysis_completion = ['Initial', 'Ongoing', 'Complete']
|
analysis_completion = ['Initial', 'Ongoing', 'Complete']
|
||||||
smtp_from = 'INSERT_FROM'
|
smtp_from = 'INSERT_FROM'
|
||||||
smtp_to = 'INSERT_TO'
|
smtp_to = 'INSERT_TO'
|
||||||
smtp_server = 'localhost'
|
smtp_server = 'localhost'
|
||||||
|
|
||||||
if args.mailoptions:
|
if args.mailoptions:
|
||||||
mailoptions = args.mailoptions.split(';')
|
mailoptions = args.mailoptions.split(';')
|
||||||
for s in mailoptions:
|
for s in mailoptions:
|
||||||
if s.split('=')[0] == 'smtp_from':
|
if s.split('=')[0] == 'smtp_from':
|
||||||
smtp_from = s.split('=')[1]
|
smtp_from = s.split('=')[1]
|
||||||
if s.split('=')[0] == 'smtp_to':
|
if s.split('=')[0] == 'smtp_to':
|
||||||
smtp_to = s.split('=')[1]
|
smtp_to = s.split('=')[1]
|
||||||
if s.split('=')[0] == 'smtp_server':
|
if s.split('=')[0] == 'smtp_server':
|
||||||
smtp_server = s.split('=')[1]
|
smtp_server = s.split('=')[1]
|
||||||
|
|
||||||
report = get_data(misp, timeframe, date_from, date_to)
|
report = get_data(misp, timeframe, date_from, date_to)
|
||||||
if(report):
|
if(report):
|
||||||
report_body, attachments = build_report(report, timeframe, misp_url)
|
report_body, attachments = build_report(report, timeframe, misp_url)
|
||||||
print_report(report_body, attachments, smtp_from, smtp_to, smtp_server, misp_url)
|
print_report(report_body, attachments, smtp_from, smtp_to, smtp_server, misp_url)
|
||||||
|
|
|
||||||
|
|
@ -54,6 +54,7 @@
|
||||||
"hex",
|
"hex",
|
||||||
"impfuzzy",
|
"impfuzzy",
|
||||||
"imphash",
|
"imphash",
|
||||||
|
"kusto-query",
|
||||||
"malware-sample",
|
"malware-sample",
|
||||||
"md5",
|
"md5",
|
||||||
"mime-type",
|
"mime-type",
|
||||||
|
|
@ -779,6 +780,10 @@
|
||||||
"default_category": "Social network",
|
"default_category": "Social network",
|
||||||
"to_ids": 0
|
"to_ids": 0
|
||||||
},
|
},
|
||||||
|
"kusto-query": {
|
||||||
|
"default_category": "Artifacts dropped",
|
||||||
|
"to_ids": 0
|
||||||
|
},
|
||||||
"last-name": {
|
"last-name": {
|
||||||
"default_category": "Person",
|
"default_category": "Person",
|
||||||
"to_ids": 0
|
"to_ids": 0
|
||||||
|
|
@ -1183,6 +1188,7 @@
|
||||||
"issue-date-of-the-visa",
|
"issue-date-of-the-visa",
|
||||||
"ja3-fingerprint-md5",
|
"ja3-fingerprint-md5",
|
||||||
"jabber-id",
|
"jabber-id",
|
||||||
|
"kusto-query",
|
||||||
"last-name",
|
"last-name",
|
||||||
"link",
|
"link",
|
||||||
"mac-address",
|
"mac-address",
|
||||||
|
|
|
||||||
|
|
@ -1 +1 @@
|
||||||
Subproject commit bce101832597b5f62679ac0299b2b4ef4681a145
|
Subproject commit ce80fb6384d6a369d4327db045255bd35bc25dbb
|
||||||
Loading…
Reference in New Issue