diff --git a/examples/treemap/README.md b/examples/treemap/README.md new file mode 100644 index 0000000..bfcc99f --- /dev/null +++ b/examples/treemap/README.md @@ -0,0 +1,11 @@ +## Explanation + +* treemap.py is a script that will generate an interactive svg (attribute\_treemap.svg) containing a treepmap representing the distribution of attributes in a sample (data) fetched from the instance using "last" or "searchall" examples. +* It will also generate a html document with a table (attribute\_table.html) containing count for each type of attribute. +* test\_attribute\_treemap.html is a quick page made to visualize both treemap and table at the same time. + +## Requierements + +* [Pygal](https://github.com/Kozea/pygal/) + + diff --git a/examples/treemap/style.css b/examples/treemap/style.css new file mode 100644 index 0000000..8c5313b --- /dev/null +++ b/examples/treemap/style.css @@ -0,0 +1,46 @@ +body +{ + /*font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;*/ + font-family: Consolas, "Liberation Mono", Menlo, Courier, monospace; +} + +h1 +{ + font-size: 16px; + width: 290px; + text-align:center; +} + +/*** Stats Tables ***/ + +table +{ + border-collapse: collapse; + border-spacing: 0; + border: 1px solid #cbcbcb; +} + +tbody +{ + font-size:12px; +} + +table td +{ + border-left: 1px solid #cbcbcb; + border-width: 0 0 0 1px; + width: 150px; + margin: 0; + padding: 0.5em 1em; +} + + +table tr:nth-child(2n-1) td +{ + background-color: #f2f2f2; +} + +table tr td:first-child +{ + font-weight: bold; +} diff --git a/examples/treemap/test_attribute_treemap.html b/examples/treemap/test_attribute_treemap.html new file mode 100644 index 0000000..d6e8fc4 --- /dev/null +++ b/examples/treemap/test_attribute_treemap.html @@ -0,0 +1,26 @@ + + + + + + + + +
+ + + diff --git a/examples/treemap/tools.py b/examples/treemap/tools.py new file mode 100644 index 0000000..1b58bca --- /dev/null +++ b/examples/treemap/tools.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import json +from json import JSONDecoder +import random +import pygal +from pygal.style import Style +import pandas as pd + +################ Formatting ################ + +def eventsListBuildFromList(filename): + with open('testt', 'r') as myfile: + s=myfile.read().replace('\n', '') + decoder = JSONDecoder() + s_len = len(s) + Events = [] + end = 0 + while end != s_len: + Event, end = decoder.raw_decode(s, idx=end) + Events.append(Event) + data = [] + for e in Events: + data.append(pd.DataFrame.from_dict(e, orient='index')) + Events = pd.concat(data) + for it in range(Events['attribute_count'].size): + if Events['attribute_count'][it] == None: + Events['attribute_count'][it]='0' + else: + Events['attribute_count'][it]=int(Events['attribute_count'][it]) + Events = Events.set_index('id') + return Events + +def eventsListBuildFromArray(filename): + ''' + returns a structure listing all primary events in the sample + ''' + jdata = json.load(open(filename)) + jdata = jdata['response'] + Events = [] + for e in jdata: + Events.append(e) + data = [] + for e in Events: + data.append(pd.DataFrame.from_dict(e, orient='index')) + Events = pd.concat(data) + for it in range(Events['attribute_count'].size): + if Events['attribute_count'][it] == None: + Events['attribute_count'][it]='0' + else: + Events['attribute_count'][it]=int(Events['attribute_count'][it]) + Events = Events.set_index('id') + return Events + +def attributesListBuild(Events): + Attributes = [] + for Attribute in Events['Attribute']: + Attributes.append(pd.DataFrame(Attribute)) + return pd.concat(Attributes) + + +################ Basic Stats ################ + +def getNbAttributePerEventCategoryType(Attributes): + return Attributes.groupby(['event_id', 'category', 'type']).count()['id'] + + +################ Charts ################ + +def createStyle(indexlevels): + colorsList = [] + for i in range(len(indexlevels[0])): + colorsList.append("#%06X" % random.randint(0, 0xFFFFFF)) + style = Style( + background='transparent', + plot_background='#FFFFFF', + foreground='#111111', + foreground_strong='#111111', + foreground_subtle='#111111', + opacity='.6', + opacity_hover='.9', + transition='400ms ease-in', + colors=tuple(colorsList)) + return style, colorsList + +def createLabelsTreemap(indexlevels, indexlabels): + categories_levels = indexlevels[0] + cat = 0 + types = [] + cattypes = [] + categories_labels = indexlabels[0] + types_levels = indexlevels[1] + types_labels = indexlabels[1] + + for it in range(len(indexlabels[0])): + if categories_labels[it] != cat: + cattypes.append(types) + types = [] + cat += 1 + + types.append(types_levels[types_labels[it]]) + cattypes.append(types) + + return categories_levels, cattypes + + +def createTable(data, title, tablename, colorsList): + if tablename == None: + target = open('attribute_table.html', 'w') + else: + target = open(tablename, 'w') + target.truncate() + target.write('\n\n\n\n\n') + categories, types = createLabelsTreemap(data.index.levels, data.index.labels) + it = 0 + + for i in range(len(categories)): + table = pygal.Treemap(pretty_print=True) + target.write('\n

' + categories[i] + '

\n') + for typ in types[i]: + table.add(typ, data[it]) + it += 1 + target.write(table.render_table(transpose=True)) + target.write('\n\n') + target.close() + + +def createTreemap(data, title, treename = 'attribute_treemap.svg', tablename = 'attribute_table.html'): + style, colorsList = createStyle(data.index.levels) + treemap = pygal.Treemap(pretty_print=True, legend_at_bottom=True, style = style) + treemap.title = title + treemap.print_values = True + treemap.print_labels = True + + categories, types = createLabelsTreemap(data.index.levels, data.index.labels) + it = 0 + + for i in range(len(categories)): + types_labels = [] + for typ in types[i]: + tempdict = {} + tempdict['label'] = typ + tempdict['value'] = data[it] + types_labels.append(tempdict) + it += 1 + treemap.add(categories[i], types_labels) + + createTable(data, 'Attribute Distribution', tablename, colorsList) + if treename == None: + treemap.render_to_file('attribute_treemap.svg') + else: + treemap.render_to_file(treename) diff --git a/examples/treemap/treemap.py b/examples/treemap/treemap.py new file mode 100644 index 0000000..1e7ef63 --- /dev/null +++ b/examples/treemap/treemap.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pymisp import PyMISP +from keys import misp_url, misp_key, misp_verifycert +import argparse +import os +import json +import tools + +def init(url, key): + return PyMISP(url, key, misp_verifycert, 'json') + +########## fetch data ########## + +def searchall(m, search, url): + result = m.search_all(search) + with open('data', 'w') as f: + f.write(json.dumps(result)) + +def download_last(m, last): + result = m.download_last(last) + with open('data', 'w') as f: + f.write(json.dumps(result)) + + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py of searchall.py) and create a treemap epresenting the distribution of attributes in this sample.') + parser.add_argument("-f", "--function", required=True, help="The parameter can be either set to \"last\" or \"searchall\". If the parameter is not valid, \"last\" will be the default setting.") + parser.add_argument("-a", "--argument", required=True, help="if function is \"last\", time can be defined in days, hours, minutes (for example 5d or 12h or 30m). Otherwise, this argument is the string to search") + + args = parser.parse_args() + + misp = init(misp_url, misp_key) + + if args.function == "searchall": + searchall(misp, args.argument, misp_url) + else: + download_last(misp, args.argument) + + Events = tools.eventsListBuildFromArray('data') + + Attributes = tools.attributesListBuild(Events) + temp = tools.getNbAttributePerEventCategoryType(Attributes) + temp = temp.groupby(level=['category', 'type']).sum() + tools.createTreemap(temp, 'Attributes Distribution', 'attribute_treemap.svg', 'attribute_table.html') +