Merge pull request #2 from Delta-Sierra/newbranch

pull/23/head
Deborah Servili 2016-07-13 16:02:04 +02:00 committed by GitHub
commit 817b38a918
11 changed files with 248 additions and 7 deletions

0
examples/events/create_dummy_event.py Normal file → Executable file
View File

6
examples/events/create_massive_dummy_events.py Normal file → Executable file
View File

@ -2,12 +2,10 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from pymisp import PyMISP from pymisp import PyMISP
from keys import misp_url, misp_key, misp_verifycert from keys import url, key
import argparse import argparse
import tools import tools
def init(url, key):
return PyMISP(url, key, misp_verifycert, 'json')
if __name__ == '__main__': if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Create a given number of event containing a given number of attributes eachh.') parser = argparse.ArgumentParser(description='Create a given number of event containing a given number of attributes eachh.')
@ -15,7 +13,7 @@ if __name__ == '__main__':
parser.add_argument("-a", "--attribute", type=int, help="Number of attributes per event (default 3000)") parser.add_argument("-a", "--attribute", type=int, help="Number of attributes per event (default 3000)")
args = parser.parse_args() args = parser.parse_args()
misp = init(misp_url, misp_key) misp = PyMISP(url, key, True, 'json')
if args.limit is None: if args.limit is None:
args.limit = 1 args.limit = 1

View File

@ -7,5 +7,3 @@
## Requierements ## Requierements
* [Pygal](https://github.com/Kozea/pygal/) * [Pygal](https://github.com/Kozea/pygal/)

View File

@ -1,5 +1,6 @@
body body
{ {
/*font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;*/
font-family: Consolas, "Liberation Mono", Menlo, Courier, monospace; font-family: Consolas, "Liberation Mono", Menlo, Courier, monospace;
} }
@ -10,6 +11,8 @@ h1
text-align:center; text-align:center;
} }
/*** Stats Tables ***/
table table
{ {
border-collapse: collapse; border-collapse: collapse;

View File

@ -0,0 +1,69 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from pymisp import PyMISP
from keys import misp_url, misp_key, misp_verifycert
from datetime import datetime
import argparse
import json
import tools
def init(url, key):
return PyMISP(url, key, misp_verifycert, 'json')
########## fetch data ##########
def searchall(m, search, url):
result = m.search_all(search)
with open('data', 'w') as f:
f.write(json.dumps(result))
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py of searchall.py) and create a treemap epresenting the distribution of attributes in this sample.')
parser.add_argument("-s", "--search", help="string to search")
parser.add_argument("-t", "--tag", required=True, help="String to search in tags, can be composed. Example: \"ransomware|Ransomware\"")
parser.add_argument("-b", "--begindate", help="The research will look for Tags attached to events posted at or after the given startdate (format: yyyy-mm-dd): If no date is given, default time is epoch time (1970-1-1)")
parser.add_argument("-e", "--enddate", help="The research will look for Tags attached to events posted at or before the given enddate (format: yyyy-mm-dd): If no date is given, default time is now()")
args = parser.parse_args()
misp = init(misp_url, misp_key)
searchall(misp, args.search, misp_url)
if args.begindate is not None:
args.begindate = tools.toDatetime(args.begindate)
if args.enddate is not None:
args.enddate = tools.toDatetime(args.enddate)
Events = tools.eventsListBuildFromArray('data')
TotalEvents = tools.getNbitems(Events)
Tags = tools.tagsListBuild(Events)
result = tools.isTagIn(Tags, args.tag)
TotalTags = len(result)
Events = tools.selectInRange(Events, begin=args.begindate, end=args.enddate)
TotalPeriodEvents = tools.getNbitems(Events)
Tags = tools.tagsListBuild(Events)
result = tools.isTagIn(Tags, args.tag)
TotalPeriodTags = len(result)
text = 'Studied pediod: from '
if args.begindate is None:
text = text + '1970-01-01'
else:
text = text + str(args.begindate.date())
text = text + ' to '
if args.enddate is None:
text = text + str(datetime.now().date())
else:
text = text + str(args.enddate.date())
print '\n========================================================'
print text
print 'During the studied pediod, ' + str(TotalPeriodTags) + ' events out of ' + str(TotalPeriodEvents) + ' contains at least one tag with ' + args.tag + '.'
if TotalTags != 0:
print 'It represents ' + str(round(100*TotalPeriodTags/TotalTags, 3)) + '% of the fetched events (' + str(TotalTags) + ') including this tag.'
if TotalEvents != 0:
print 'It also represents ' + str(round(100*TotalPeriodTags/TotalEvents, 3)) + '% of all the fetched events (' + str(TotalEvents) + ').'

View File

@ -0,0 +1,70 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from pymisp import PyMISP
from keys import misp_url, misp_key, misp_verifycert
from datetime import datetime
import argparse
import json
import tools
def init(url, key):
return PyMISP(url, key, misp_verifycert, 'json')
########## fetch data ##########
def searchall(m, search, url):
result = m.search_all(search)
with open('data', 'w') as f:
f.write(json.dumps(result))
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py of searchall.py) and create a treemap epresenting the distribution of attributes in this sample.')
parser.add_argument("-s", "--search", help="string to search")
parser.add_argument("-b", "--begindate", help="The research will look for Tags attached to events posted at or after the given startdate (format: yyyy-mm-dd): If no date is given, default time is epoch time (1970-1-1)")
parser.add_argument("-e", "--enddate", help="The research will look for Tags attached to events posted at or before the given enddate (format: yyyy-mm-dd): If no date is given, default time is now()")
args = parser.parse_args()
misp = init(misp_url, misp_key)
if args.search is None:
args.search = ''
searchall(misp, args.search, misp_url)
if args.begindate is not None:
args.begindate = tools.toDatetime(args.begindate)
if args.enddate is not None:
args.enddate = tools.toDatetime(args.enddate)
Events = tools.eventsListBuildFromArray('data')
TotalEvents = tools.getNbitems(Events)
Tags = tools.tagsListBuild(Events)
result = tools.getNbOccurenceTags(Tags)
TotalTags = tools.getNbitems(Tags)
Events = tools.selectInRange(Events, begin=args.begindate, end=args.enddate)
TotalPeriodEvents = tools.getNbitems(Events)
Tags = tools.tagsListBuild(Events)
result = tools.getNbOccurenceTags(Tags)
TotalPeriodTags = tools.getNbitems(Tags)
text = 'Studied pediod: from '
if args.begindate is None:
text = text + '1970-01-01'
else:
text = text + str(args.begindate.date())
text = text + ' to '
if args.enddate is None:
text = text + str(datetime.now().date())
else:
text = text + str(args.enddate.date())
print '\n========================================================'
print text
print result
'''
print 'During the studied pediod, ' + str(TotalPeriodTags) + ' events out of ' + str(TotalPeriodEvents) + ' contains at least one tag with ' + args.tag + '.'
print 'It represents ' + str(round(100*TotalPeriodTags/TotalTags,3)) + '% of the fetched events (' + str(TotalTags) + ') including this tag.'
print 'It also represents ' + str(round(100*TotalPeriodTags/TotalEvents,3)) + '% of all the fetched events (' + str(TotalEvents) + ').'
'''

View File

@ -18,5 +18,9 @@
<td><iframe id="stats" src="attribute_table.html" frameBorder="0"></iframe></td> <td><iframe id="stats" src="attribute_table.html" frameBorder="0"></iframe></td>
<td id="treemap"><object type="image/svg+xml" data="attribute_treemap.svg"></object></td> <td id="treemap"><object type="image/svg+xml" data="attribute_treemap.svg"></object></td>
</tr></table> </tr></table>
<!--
<div id="stats"><iframe src="table.html"></iframe></div>
<div id="treemap"><object type="image/svg+xml" data="test.svg"></object></div>
-->
</body> </body>
</html> </html>

View File

@ -7,6 +7,52 @@ import random
import pygal import pygal
from pygal.style import Style from pygal.style import Style
import pandas as pd import pandas as pd
from datetime import datetime
from datetime import timedelta
from dateutil.parser import parse
################ Tools ################
def buildDoubleIndex(index1, index2, datatype):
it = -1
newindex1 = []
for index in index2:
if index == 0:
it+=1
newindex1.append(index1[it])
arrays = [newindex1, index2]
tuples = list(zip(*arrays))
return pd.MultiIndex.from_tuples(tuples, names=['event', datatype])
def buildNewColumn(index2, column):
it = -1
newcolumn = []
for index in index2:
if index == 0:
it+=1
newcolumn.append(column[it])
return newcolumn
def dateInRange(datetimeTested, begin=None, end=None):
if begin == None:
begin = datetime(1970,1,1)
if end == None:
end = datetime.now()
return begin <= datetimeTested <= end
def addColumn(dataframe, columnList, columnName):
dataframe.loc[:, columnName] = pd.Series(columnList, index=dataframe.index)
def dateInRange(datetimeTested, begin=None, end=None):
if begin == None:
begin = datetime(1970,1,1)
if end == None:
end = datetime.now()
return begin <= datetimeTested <= end
def toDatetime(date):
temp = date.split('-')
return datetime(int(temp[0]), int(temp[1]), int(temp[2]))
################ Formatting ################ ################ Formatting ################
@ -59,12 +105,58 @@ def attributesListBuild(Events):
Attributes.append(pd.DataFrame(Attribute)) Attributes.append(pd.DataFrame(Attribute))
return pd.concat(Attributes) return pd.concat(Attributes)
def tagsListBuild(Events):
Tags = []
for Tag in Events['Tag']:
if type(Tag) is not list:
continue
Tags.append(pd.DataFrame(Tag))
Tags = pd.concat(Tags)
columnDate = buildNewColumn(Tags.index, Events['date'])
addColumn(Tags, columnDate, 'date')
index = buildDoubleIndex(Events.index, Tags.index, 'tag')
Tags = Tags.set_index(index)
return Tags
def selectInRange(Events, begin=None, end=None):
inRange = []
for i, Event in Events.iterrows():
if dateInRange(parse(Event['date']), begin, end):
inRange.append(Event.tolist())
inRange = pd.DataFrame(inRange)
temp = Events.columns.tolist()
inRange.columns = temp
return inRange
'''
def isTagIn(dataframe, tag):
print 'tag =' + tag
result = []
for tagname in dataframe['name']:
print tagname
if tag in tagname:
print 'True'
result.append(tagname)
return result
'''
def isTagIn(dataframe, tag):
temp = Tags[Tags['name'].str.contains(test)].index.tolist()
index = []
for i in range(len(temp)):
if temp[i][0] not in index:
index.append(temp[i][0])
return index
################ Basic Stats ################ ################ Basic Stats ################
def getNbitems(dataframe):
return len(dataframe.index)
def getNbAttributePerEventCategoryType(Attributes): def getNbAttributePerEventCategoryType(Attributes):
return Attributes.groupby(['event_id', 'category', 'type']).count()['id'] return Attributes.groupby(['event_id', 'category', 'type']).count()['id']
def getNbOccurenceTags(Tags):
return Tags.groupby('name').count()['id']
################ Charts ################ ################ Charts ################

View File

@ -1,3 +1,3 @@
__version__ = '2.4.48.1' __version__ = '2.4.48.2'
from .api import PyMISP, PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey from .api import PyMISP, PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey

View File

@ -364,6 +364,13 @@ class PyMISP(object):
return self._check_response(response) return self._check_response(response)
def remove_tag(self, event, tag):
session = self.__prepare_session('json')
to_post = {'request': {'Event': {'id': event['Event']['id'], 'tag': tag}}}
response = session.post(urljoin(self.root_url, 'events/removeTag'), data=json.dumps(to_post))
return self._check_response(response)
def change_threat_level(self, event, threat_level_id): def change_threat_level(self, event, threat_level_id):
event['Event']['threat_level_id'] = threat_level_id event['Event']['threat_level_id'] = threat_level_id
self._prepare_update(event) self._prepare_update(event)