mirror of https://github.com/MISP/PyMISP
delete some files
commit
eebca6ecc6
|
@ -1,11 +0,0 @@
|
||||||
## Explanation
|
|
||||||
|
|
||||||
* treemap.py is a script that will generate an interactive svg (attribute\_treemap.svg) containing a treepmap representing the distribution of attributes in a sample (data) fetched from the instance using "last" or "searchall" examples.
|
|
||||||
* It will also generate a html document with a table (attribute\_table.html) containing count for each type of attribute.
|
|
||||||
* test\_attribute\_treemap.html is a quick page made to visualize both treemap and table at the same time.
|
|
||||||
|
|
||||||
## Requierements
|
|
||||||
|
|
||||||
* [Pygal](https://github.com/Kozea/pygal/)
|
|
||||||
|
|
||||||
|
|
|
@ -1,43 +0,0 @@
|
||||||
body
|
|
||||||
{
|
|
||||||
font-family: Consolas, "Liberation Mono", Menlo, Courier, monospace;
|
|
||||||
}
|
|
||||||
|
|
||||||
h1
|
|
||||||
{
|
|
||||||
font-size: 16px;
|
|
||||||
width: 290px;
|
|
||||||
text-align:center;
|
|
||||||
}
|
|
||||||
|
|
||||||
table
|
|
||||||
{
|
|
||||||
border-collapse: collapse;
|
|
||||||
border-spacing: 0;
|
|
||||||
border: 1px solid #cbcbcb;
|
|
||||||
}
|
|
||||||
|
|
||||||
tbody
|
|
||||||
{
|
|
||||||
font-size:12px;
|
|
||||||
}
|
|
||||||
|
|
||||||
table td
|
|
||||||
{
|
|
||||||
border-left: 1px solid #cbcbcb;
|
|
||||||
border-width: 0 0 0 1px;
|
|
||||||
width: 150px;
|
|
||||||
margin: 0;
|
|
||||||
padding: 0.5em 1em;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
table tr:nth-child(2n-1) td
|
|
||||||
{
|
|
||||||
background-color: #f2f2f2;
|
|
||||||
}
|
|
||||||
|
|
||||||
table tr td:first-child
|
|
||||||
{
|
|
||||||
font-weight: bold;
|
|
||||||
}
|
|
|
@ -1,22 +0,0 @@
|
||||||
<html>
|
|
||||||
<head>
|
|
||||||
<style>
|
|
||||||
#stats
|
|
||||||
{
|
|
||||||
height: 746px;
|
|
||||||
margin-top: 100px;
|
|
||||||
}
|
|
||||||
|
|
||||||
#treemap
|
|
||||||
{
|
|
||||||
width: 1000px;
|
|
||||||
}
|
|
||||||
</style>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<table><tr>
|
|
||||||
<td><iframe id="stats" src="attribute_table.html" frameBorder="0"></iframe></td>
|
|
||||||
<td id="treemap"><object type="image/svg+xml" data="attribute_treemap.svg"></object></td>
|
|
||||||
</tr></table>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
|
@ -1,153 +0,0 @@
|
||||||
#!/usr/bin/env python
|
|
||||||
# -*- coding: utf-8 -*-
|
|
||||||
|
|
||||||
import json
|
|
||||||
from json import JSONDecoder
|
|
||||||
import random
|
|
||||||
import pygal
|
|
||||||
from pygal.style import Style
|
|
||||||
import pandas as pd
|
|
||||||
|
|
||||||
################ Formatting ################
|
|
||||||
|
|
||||||
def eventsListBuildFromList(filename):
|
|
||||||
with open('testt', 'r') as myfile:
|
|
||||||
s=myfile.read().replace('\n', '')
|
|
||||||
decoder = JSONDecoder()
|
|
||||||
s_len = len(s)
|
|
||||||
Events = []
|
|
||||||
end = 0
|
|
||||||
while end != s_len:
|
|
||||||
Event, end = decoder.raw_decode(s, idx=end)
|
|
||||||
Events.append(Event)
|
|
||||||
data = []
|
|
||||||
for e in Events:
|
|
||||||
data.append(pd.DataFrame.from_dict(e, orient='index'))
|
|
||||||
Events = pd.concat(data)
|
|
||||||
for it in range(Events['attribute_count'].size):
|
|
||||||
if Events['attribute_count'][it] == None:
|
|
||||||
Events['attribute_count'][it]='0'
|
|
||||||
else:
|
|
||||||
Events['attribute_count'][it]=int(Events['attribute_count'][it])
|
|
||||||
Events = Events.set_index('id')
|
|
||||||
return Events
|
|
||||||
|
|
||||||
def eventsListBuildFromArray(filename):
|
|
||||||
'''
|
|
||||||
returns a structure listing all primary events in the sample
|
|
||||||
'''
|
|
||||||
jdata = json.load(open(filename))
|
|
||||||
jdata = jdata['response']
|
|
||||||
Events = []
|
|
||||||
for e in jdata:
|
|
||||||
Events.append(e)
|
|
||||||
data = []
|
|
||||||
for e in Events:
|
|
||||||
data.append(pd.DataFrame.from_dict(e, orient='index'))
|
|
||||||
Events = pd.concat(data)
|
|
||||||
for it in range(Events['attribute_count'].size):
|
|
||||||
if Events['attribute_count'][it] == None:
|
|
||||||
Events['attribute_count'][it]='0'
|
|
||||||
else:
|
|
||||||
Events['attribute_count'][it]=int(Events['attribute_count'][it])
|
|
||||||
Events = Events.set_index('id')
|
|
||||||
return Events
|
|
||||||
|
|
||||||
def attributesListBuild(Events):
|
|
||||||
Attributes = []
|
|
||||||
for Attribute in Events['Attribute']:
|
|
||||||
Attributes.append(pd.DataFrame(Attribute))
|
|
||||||
return pd.concat(Attributes)
|
|
||||||
|
|
||||||
|
|
||||||
################ Basic Stats ################
|
|
||||||
|
|
||||||
def getNbAttributePerEventCategoryType(Attributes):
|
|
||||||
return Attributes.groupby(['event_id', 'category', 'type']).count()['id']
|
|
||||||
|
|
||||||
|
|
||||||
################ Charts ################
|
|
||||||
|
|
||||||
def createStyle(indexlevels):
|
|
||||||
colorsList = []
|
|
||||||
for i in range(len(indexlevels[0])):
|
|
||||||
colorsList.append("#%06X" % random.randint(0, 0xFFFFFF))
|
|
||||||
style = Style(
|
|
||||||
background='transparent',
|
|
||||||
plot_background='#FFFFFF',
|
|
||||||
foreground='#111111',
|
|
||||||
foreground_strong='#111111',
|
|
||||||
foreground_subtle='#111111',
|
|
||||||
opacity='.6',
|
|
||||||
opacity_hover='.9',
|
|
||||||
transition='400ms ease-in',
|
|
||||||
colors=tuple(colorsList))
|
|
||||||
return style, colorsList
|
|
||||||
|
|
||||||
def createLabelsTreemap(indexlevels, indexlabels):
|
|
||||||
categories_levels = indexlevels[0]
|
|
||||||
cat = 0
|
|
||||||
types = []
|
|
||||||
cattypes = []
|
|
||||||
categories_labels = indexlabels[0]
|
|
||||||
types_levels = indexlevels[1]
|
|
||||||
types_labels = indexlabels[1]
|
|
||||||
|
|
||||||
for it in range(len(indexlabels[0])):
|
|
||||||
if categories_labels[it] != cat:
|
|
||||||
cattypes.append(types)
|
|
||||||
types = []
|
|
||||||
cat += 1
|
|
||||||
|
|
||||||
types.append(types_levels[types_labels[it]])
|
|
||||||
cattypes.append(types)
|
|
||||||
|
|
||||||
return categories_levels, cattypes
|
|
||||||
|
|
||||||
|
|
||||||
def createTable(data, title, tablename, colorsList):
|
|
||||||
if tablename == None:
|
|
||||||
target = open('attribute_table.html', 'w')
|
|
||||||
else:
|
|
||||||
target = open(tablename, 'w')
|
|
||||||
target.truncate()
|
|
||||||
target.write('<!DOCTYPE html>\n<html>\n<head>\n<link rel="stylesheet" href="style.css">\n</head>\n<body>')
|
|
||||||
categories, types = createLabelsTreemap(data.index.levels, data.index.labels)
|
|
||||||
it = 0
|
|
||||||
|
|
||||||
for i in range(len(categories)):
|
|
||||||
table = pygal.Treemap(pretty_print=True)
|
|
||||||
target.write('\n <h1 style="color:'+ colorsList[i]+ ';">' + categories[i] + '</h1>\n')
|
|
||||||
for typ in types[i]:
|
|
||||||
table.add(typ, data[it])
|
|
||||||
it += 1
|
|
||||||
target.write(table.render_table(transpose=True))
|
|
||||||
target.write('\n</body>\n</html>')
|
|
||||||
target.close()
|
|
||||||
|
|
||||||
|
|
||||||
def createTreemap(data, title, treename = 'attribute_treemap.svg', tablename = 'attribute_table.html'):
|
|
||||||
style, colorsList = createStyle(data.index.levels)
|
|
||||||
treemap = pygal.Treemap(pretty_print=True, legend_at_bottom=True, style = style)
|
|
||||||
treemap.title = title
|
|
||||||
treemap.print_values = True
|
|
||||||
treemap.print_labels = True
|
|
||||||
|
|
||||||
categories, types = createLabelsTreemap(data.index.levels, data.index.labels)
|
|
||||||
it = 0
|
|
||||||
|
|
||||||
for i in range(len(categories)):
|
|
||||||
types_labels = []
|
|
||||||
for typ in types[i]:
|
|
||||||
tempdict = {}
|
|
||||||
tempdict['label'] = typ
|
|
||||||
tempdict['value'] = data[it]
|
|
||||||
types_labels.append(tempdict)
|
|
||||||
it += 1
|
|
||||||
treemap.add(categories[i], types_labels)
|
|
||||||
|
|
||||||
createTable(data, 'Attribute Distribution', tablename, colorsList)
|
|
||||||
if treename == None:
|
|
||||||
treemap.render_to_file('attribute_treemap.svg')
|
|
||||||
else:
|
|
||||||
treemap.render_to_file(treename)
|
|
Loading…
Reference in New Issue