mirror of https://github.com/MISP/PyMISP
Merge pull request #2 from Delta-Sierra/newbranch
commit
817b38a918
|
@ -2,12 +2,10 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
from pymisp import PyMISP
|
from pymisp import PyMISP
|
||||||
from keys import misp_url, misp_key, misp_verifycert
|
from keys import url, key
|
||||||
import argparse
|
import argparse
|
||||||
import tools
|
import tools
|
||||||
|
|
||||||
def init(url, key):
|
|
||||||
return PyMISP(url, key, misp_verifycert, 'json')
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
parser = argparse.ArgumentParser(description='Create a given number of event containing a given number of attributes eachh.')
|
parser = argparse.ArgumentParser(description='Create a given number of event containing a given number of attributes eachh.')
|
||||||
|
@ -15,7 +13,7 @@ if __name__ == '__main__':
|
||||||
parser.add_argument("-a", "--attribute", type=int, help="Number of attributes per event (default 3000)")
|
parser.add_argument("-a", "--attribute", type=int, help="Number of attributes per event (default 3000)")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
misp = init(misp_url, misp_key)
|
misp = PyMISP(url, key, True, 'json')
|
||||||
|
|
||||||
if args.limit is None:
|
if args.limit is None:
|
||||||
args.limit = 1
|
args.limit = 1
|
||||||
|
|
|
@ -7,5 +7,3 @@
|
||||||
## Requierements
|
## Requierements
|
||||||
|
|
||||||
* [Pygal](https://github.com/Kozea/pygal/)
|
* [Pygal](https://github.com/Kozea/pygal/)
|
||||||
|
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
body
|
body
|
||||||
{
|
{
|
||||||
|
/*font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;*/
|
||||||
font-family: Consolas, "Liberation Mono", Menlo, Courier, monospace;
|
font-family: Consolas, "Liberation Mono", Menlo, Courier, monospace;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -10,6 +11,8 @@ h1
|
||||||
text-align:center;
|
text-align:center;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*** Stats Tables ***/
|
||||||
|
|
||||||
table
|
table
|
||||||
{
|
{
|
||||||
border-collapse: collapse;
|
border-collapse: collapse;
|
|
@ -0,0 +1,69 @@
|
||||||
|
#!/usr/bin/env python
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
from pymisp import PyMISP
|
||||||
|
from keys import misp_url, misp_key, misp_verifycert
|
||||||
|
from datetime import datetime
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import tools
|
||||||
|
|
||||||
|
def init(url, key):
|
||||||
|
return PyMISP(url, key, misp_verifycert, 'json')
|
||||||
|
|
||||||
|
########## fetch data ##########
|
||||||
|
|
||||||
|
def searchall(m, search, url):
|
||||||
|
result = m.search_all(search)
|
||||||
|
with open('data', 'w') as f:
|
||||||
|
f.write(json.dumps(result))
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py of searchall.py) and create a treemap epresenting the distribution of attributes in this sample.')
|
||||||
|
parser.add_argument("-s", "--search", help="string to search")
|
||||||
|
parser.add_argument("-t", "--tag", required=True, help="String to search in tags, can be composed. Example: \"ransomware|Ransomware\"")
|
||||||
|
parser.add_argument("-b", "--begindate", help="The research will look for Tags attached to events posted at or after the given startdate (format: yyyy-mm-dd): If no date is given, default time is epoch time (1970-1-1)")
|
||||||
|
parser.add_argument("-e", "--enddate", help="The research will look for Tags attached to events posted at or before the given enddate (format: yyyy-mm-dd): If no date is given, default time is now()")
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
misp = init(misp_url, misp_key)
|
||||||
|
|
||||||
|
searchall(misp, args.search, misp_url)
|
||||||
|
|
||||||
|
if args.begindate is not None:
|
||||||
|
args.begindate = tools.toDatetime(args.begindate)
|
||||||
|
if args.enddate is not None:
|
||||||
|
args.enddate = tools.toDatetime(args.enddate)
|
||||||
|
|
||||||
|
Events = tools.eventsListBuildFromArray('data')
|
||||||
|
TotalEvents = tools.getNbitems(Events)
|
||||||
|
Tags = tools.tagsListBuild(Events)
|
||||||
|
result = tools.isTagIn(Tags, args.tag)
|
||||||
|
TotalTags = len(result)
|
||||||
|
|
||||||
|
Events = tools.selectInRange(Events, begin=args.begindate, end=args.enddate)
|
||||||
|
TotalPeriodEvents = tools.getNbitems(Events)
|
||||||
|
Tags = tools.tagsListBuild(Events)
|
||||||
|
result = tools.isTagIn(Tags, args.tag)
|
||||||
|
TotalPeriodTags = len(result)
|
||||||
|
|
||||||
|
text = 'Studied pediod: from '
|
||||||
|
if args.begindate is None:
|
||||||
|
text = text + '1970-01-01'
|
||||||
|
else:
|
||||||
|
text = text + str(args.begindate.date())
|
||||||
|
text = text + ' to '
|
||||||
|
if args.enddate is None:
|
||||||
|
text = text + str(datetime.now().date())
|
||||||
|
else:
|
||||||
|
text = text + str(args.enddate.date())
|
||||||
|
|
||||||
|
print '\n========================================================'
|
||||||
|
print text
|
||||||
|
print 'During the studied pediod, ' + str(TotalPeriodTags) + ' events out of ' + str(TotalPeriodEvents) + ' contains at least one tag with ' + args.tag + '.'
|
||||||
|
if TotalTags != 0:
|
||||||
|
print 'It represents ' + str(round(100*TotalPeriodTags/TotalTags, 3)) + '% of the fetched events (' + str(TotalTags) + ') including this tag.'
|
||||||
|
if TotalEvents != 0:
|
||||||
|
print 'It also represents ' + str(round(100*TotalPeriodTags/TotalEvents, 3)) + '% of all the fetched events (' + str(TotalEvents) + ').'
|
||||||
|
|
|
@ -0,0 +1,70 @@
|
||||||
|
#!/usr/bin/env python
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
from pymisp import PyMISP
|
||||||
|
from keys import misp_url, misp_key, misp_verifycert
|
||||||
|
from datetime import datetime
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import tools
|
||||||
|
|
||||||
|
def init(url, key):
|
||||||
|
return PyMISP(url, key, misp_verifycert, 'json')
|
||||||
|
|
||||||
|
########## fetch data ##########
|
||||||
|
|
||||||
|
def searchall(m, search, url):
|
||||||
|
result = m.search_all(search)
|
||||||
|
with open('data', 'w') as f:
|
||||||
|
f.write(json.dumps(result))
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py of searchall.py) and create a treemap epresenting the distribution of attributes in this sample.')
|
||||||
|
parser.add_argument("-s", "--search", help="string to search")
|
||||||
|
parser.add_argument("-b", "--begindate", help="The research will look for Tags attached to events posted at or after the given startdate (format: yyyy-mm-dd): If no date is given, default time is epoch time (1970-1-1)")
|
||||||
|
parser.add_argument("-e", "--enddate", help="The research will look for Tags attached to events posted at or before the given enddate (format: yyyy-mm-dd): If no date is given, default time is now()")
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
misp = init(misp_url, misp_key)
|
||||||
|
|
||||||
|
if args.search is None:
|
||||||
|
args.search = ''
|
||||||
|
searchall(misp, args.search, misp_url)
|
||||||
|
|
||||||
|
if args.begindate is not None:
|
||||||
|
args.begindate = tools.toDatetime(args.begindate)
|
||||||
|
if args.enddate is not None:
|
||||||
|
args.enddate = tools.toDatetime(args.enddate)
|
||||||
|
|
||||||
|
Events = tools.eventsListBuildFromArray('data')
|
||||||
|
TotalEvents = tools.getNbitems(Events)
|
||||||
|
Tags = tools.tagsListBuild(Events)
|
||||||
|
result = tools.getNbOccurenceTags(Tags)
|
||||||
|
TotalTags = tools.getNbitems(Tags)
|
||||||
|
|
||||||
|
Events = tools.selectInRange(Events, begin=args.begindate, end=args.enddate)
|
||||||
|
TotalPeriodEvents = tools.getNbitems(Events)
|
||||||
|
Tags = tools.tagsListBuild(Events)
|
||||||
|
result = tools.getNbOccurenceTags(Tags)
|
||||||
|
TotalPeriodTags = tools.getNbitems(Tags)
|
||||||
|
|
||||||
|
text = 'Studied pediod: from '
|
||||||
|
if args.begindate is None:
|
||||||
|
text = text + '1970-01-01'
|
||||||
|
else:
|
||||||
|
text = text + str(args.begindate.date())
|
||||||
|
text = text + ' to '
|
||||||
|
if args.enddate is None:
|
||||||
|
text = text + str(datetime.now().date())
|
||||||
|
else:
|
||||||
|
text = text + str(args.enddate.date())
|
||||||
|
|
||||||
|
print '\n========================================================'
|
||||||
|
print text
|
||||||
|
print result
|
||||||
|
'''
|
||||||
|
print 'During the studied pediod, ' + str(TotalPeriodTags) + ' events out of ' + str(TotalPeriodEvents) + ' contains at least one tag with ' + args.tag + '.'
|
||||||
|
print 'It represents ' + str(round(100*TotalPeriodTags/TotalTags,3)) + '% of the fetched events (' + str(TotalTags) + ') including this tag.'
|
||||||
|
print 'It also represents ' + str(round(100*TotalPeriodTags/TotalEvents,3)) + '% of all the fetched events (' + str(TotalEvents) + ').'
|
||||||
|
'''
|
|
@ -18,5 +18,9 @@
|
||||||
<td><iframe id="stats" src="attribute_table.html" frameBorder="0"></iframe></td>
|
<td><iframe id="stats" src="attribute_table.html" frameBorder="0"></iframe></td>
|
||||||
<td id="treemap"><object type="image/svg+xml" data="attribute_treemap.svg"></object></td>
|
<td id="treemap"><object type="image/svg+xml" data="attribute_treemap.svg"></object></td>
|
||||||
</tr></table>
|
</tr></table>
|
||||||
|
<!--
|
||||||
|
<div id="stats"><iframe src="table.html"></iframe></div>
|
||||||
|
<div id="treemap"><object type="image/svg+xml" data="test.svg"></object></div>
|
||||||
|
-->
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
|
@ -7,6 +7,52 @@ import random
|
||||||
import pygal
|
import pygal
|
||||||
from pygal.style import Style
|
from pygal.style import Style
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
|
from datetime import datetime
|
||||||
|
from datetime import timedelta
|
||||||
|
from dateutil.parser import parse
|
||||||
|
|
||||||
|
################ Tools ################
|
||||||
|
|
||||||
|
def buildDoubleIndex(index1, index2, datatype):
|
||||||
|
it = -1
|
||||||
|
newindex1 = []
|
||||||
|
for index in index2:
|
||||||
|
if index == 0:
|
||||||
|
it+=1
|
||||||
|
newindex1.append(index1[it])
|
||||||
|
arrays = [newindex1, index2]
|
||||||
|
tuples = list(zip(*arrays))
|
||||||
|
return pd.MultiIndex.from_tuples(tuples, names=['event', datatype])
|
||||||
|
|
||||||
|
def buildNewColumn(index2, column):
|
||||||
|
it = -1
|
||||||
|
newcolumn = []
|
||||||
|
for index in index2:
|
||||||
|
if index == 0:
|
||||||
|
it+=1
|
||||||
|
newcolumn.append(column[it])
|
||||||
|
return newcolumn
|
||||||
|
|
||||||
|
def dateInRange(datetimeTested, begin=None, end=None):
|
||||||
|
if begin == None:
|
||||||
|
begin = datetime(1970,1,1)
|
||||||
|
if end == None:
|
||||||
|
end = datetime.now()
|
||||||
|
return begin <= datetimeTested <= end
|
||||||
|
|
||||||
|
def addColumn(dataframe, columnList, columnName):
|
||||||
|
dataframe.loc[:, columnName] = pd.Series(columnList, index=dataframe.index)
|
||||||
|
|
||||||
|
def dateInRange(datetimeTested, begin=None, end=None):
|
||||||
|
if begin == None:
|
||||||
|
begin = datetime(1970,1,1)
|
||||||
|
if end == None:
|
||||||
|
end = datetime.now()
|
||||||
|
return begin <= datetimeTested <= end
|
||||||
|
|
||||||
|
def toDatetime(date):
|
||||||
|
temp = date.split('-')
|
||||||
|
return datetime(int(temp[0]), int(temp[1]), int(temp[2]))
|
||||||
|
|
||||||
################ Formatting ################
|
################ Formatting ################
|
||||||
|
|
||||||
|
@ -59,12 +105,58 @@ def attributesListBuild(Events):
|
||||||
Attributes.append(pd.DataFrame(Attribute))
|
Attributes.append(pd.DataFrame(Attribute))
|
||||||
return pd.concat(Attributes)
|
return pd.concat(Attributes)
|
||||||
|
|
||||||
|
def tagsListBuild(Events):
|
||||||
|
Tags = []
|
||||||
|
for Tag in Events['Tag']:
|
||||||
|
if type(Tag) is not list:
|
||||||
|
continue
|
||||||
|
Tags.append(pd.DataFrame(Tag))
|
||||||
|
Tags = pd.concat(Tags)
|
||||||
|
columnDate = buildNewColumn(Tags.index, Events['date'])
|
||||||
|
addColumn(Tags, columnDate, 'date')
|
||||||
|
index = buildDoubleIndex(Events.index, Tags.index, 'tag')
|
||||||
|
Tags = Tags.set_index(index)
|
||||||
|
return Tags
|
||||||
|
|
||||||
|
def selectInRange(Events, begin=None, end=None):
|
||||||
|
inRange = []
|
||||||
|
for i, Event in Events.iterrows():
|
||||||
|
if dateInRange(parse(Event['date']), begin, end):
|
||||||
|
inRange.append(Event.tolist())
|
||||||
|
inRange = pd.DataFrame(inRange)
|
||||||
|
temp = Events.columns.tolist()
|
||||||
|
inRange.columns = temp
|
||||||
|
return inRange
|
||||||
|
'''
|
||||||
|
def isTagIn(dataframe, tag):
|
||||||
|
print 'tag =' + tag
|
||||||
|
result = []
|
||||||
|
for tagname in dataframe['name']:
|
||||||
|
print tagname
|
||||||
|
if tag in tagname:
|
||||||
|
print 'True'
|
||||||
|
result.append(tagname)
|
||||||
|
return result
|
||||||
|
'''
|
||||||
|
|
||||||
|
def isTagIn(dataframe, tag):
|
||||||
|
temp = Tags[Tags['name'].str.contains(test)].index.tolist()
|
||||||
|
index = []
|
||||||
|
for i in range(len(temp)):
|
||||||
|
if temp[i][0] not in index:
|
||||||
|
index.append(temp[i][0])
|
||||||
|
return index
|
||||||
|
|
||||||
################ Basic Stats ################
|
################ Basic Stats ################
|
||||||
|
|
||||||
|
def getNbitems(dataframe):
|
||||||
|
return len(dataframe.index)
|
||||||
|
|
||||||
def getNbAttributePerEventCategoryType(Attributes):
|
def getNbAttributePerEventCategoryType(Attributes):
|
||||||
return Attributes.groupby(['event_id', 'category', 'type']).count()['id']
|
return Attributes.groupby(['event_id', 'category', 'type']).count()['id']
|
||||||
|
|
||||||
|
def getNbOccurenceTags(Tags):
|
||||||
|
return Tags.groupby('name').count()['id']
|
||||||
|
|
||||||
################ Charts ################
|
################ Charts ################
|
||||||
|
|
|
@ -1,3 +1,3 @@
|
||||||
__version__ = '2.4.48.1'
|
__version__ = '2.4.48.2'
|
||||||
|
|
||||||
from .api import PyMISP, PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey
|
from .api import PyMISP, PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey
|
||||||
|
|
|
@ -364,6 +364,13 @@ class PyMISP(object):
|
||||||
|
|
||||||
return self._check_response(response)
|
return self._check_response(response)
|
||||||
|
|
||||||
|
def remove_tag(self, event, tag):
|
||||||
|
session = self.__prepare_session('json')
|
||||||
|
to_post = {'request': {'Event': {'id': event['Event']['id'], 'tag': tag}}}
|
||||||
|
response = session.post(urljoin(self.root_url, 'events/removeTag'), data=json.dumps(to_post))
|
||||||
|
|
||||||
|
return self._check_response(response)
|
||||||
|
|
||||||
def change_threat_level(self, event, threat_level_id):
|
def change_threat_level(self, event, threat_level_id):
|
||||||
event['Event']['threat_level_id'] = threat_level_id
|
event['Event']['threat_level_id'] = threat_level_id
|
||||||
self._prepare_update(event)
|
self._prepare_update(event)
|
||||||
|
|
Loading…
Reference in New Issue