Merge pull request #2 from Delta-Sierra/newbranch

examples/events/create_massive_dummy_events.py

@@ -2,12 +2,10 @@
 # -*- coding: utf-8 -*-
 
 from pymisp import PyMISP
-from keys import misp_url, misp_key, misp_verifycert
+from keys import url, key
 import argparse
 import tools
 
-def init(url, key):
-    return PyMISP(url, key, misp_verifycert, 'json')
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Create a given number of event containing a given number of attributes eachh.')
@@ -15,7 +13,7 @@ if __name__ == '__main__':
     parser.add_argument("-a", "--attribute", type=int, help="Number of attributes per event (default 3000)")
     args = parser.parse_args()
 
-    misp = init(misp_url, misp_key)
+    misp = PyMISP(url, key, True, 'json')
 
     if args.limit is None:
         args.limit = 1

examples/situational-awareness/README.md

@@ -7,5 +7,3 @@
 ## Requierements
 
 * [Pygal](https://github.com/Kozea/pygal/)
-
-

examples/situational-awareness/style.css

@@ -1,5 +1,6 @@
 body 
 {
+    /*font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;*/
 	font-family: Consolas, "Liberation Mono", Menlo, Courier, monospace;
 }
 
@@ -10,6 +11,8 @@ h1
 	text-align:center;
 }
 
+/*** Stats Tables ***/
+
 table
 {
 	border-collapse: collapse;

examples/situational-awareness/tag_search.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from pymisp import PyMISP
+from keys import misp_url, misp_key, misp_verifycert
+from datetime import datetime
+import argparse
+import json
+import tools
+
+def init(url, key):
+    return PyMISP(url, key, misp_verifycert, 'json')
+
+########## fetch data ##########
+
+def searchall(m, search, url):
+    result = m.search_all(search)
+    with open('data', 'w') as f:
+        f.write(json.dumps(result))
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py of searchall.py) and create a treemap epresenting the distribution of attributes in this sample.')
+    parser.add_argument("-s", "--search", help="string to search")
+    parser.add_argument("-t", "--tag", required=True, help="String to search in tags, can be composed. Example: \"ransomware|Ransomware\"")
+    parser.add_argument("-b", "--begindate", help="The research will look for Tags attached to events posted at or after the given startdate (format: yyyy-mm-dd): If no date is given, default time is epoch time (1970-1-1)")
+    parser.add_argument("-e", "--enddate", help="The research will look for Tags attached to events posted at or before the given enddate (format: yyyy-mm-dd): If no date is given, default time is now()")
+
+    args = parser.parse_args()
+
+    misp = init(misp_url, misp_key)
+
+    searchall(misp, args.search, misp_url)
+
+    if args.begindate is not None:
+        args.begindate = tools.toDatetime(args.begindate)
+    if args.enddate is not None:
+        args.enddate = tools.toDatetime(args.enddate)
+
+    Events = tools.eventsListBuildFromArray('data')
+    TotalEvents = tools.getNbitems(Events)
+    Tags = tools.tagsListBuild(Events)
+    result = tools.isTagIn(Tags, args.tag)
+    TotalTags = len(result)
+
+    Events = tools.selectInRange(Events, begin=args.begindate, end=args.enddate)
+    TotalPeriodEvents = tools.getNbitems(Events)
+    Tags = tools.tagsListBuild(Events)
+    result = tools.isTagIn(Tags, args.tag)
+    TotalPeriodTags = len(result)
+
+    text = 'Studied pediod: from '
+    if args.begindate is None:
+        text = text + '1970-01-01'
+    else:
+        text = text + str(args.begindate.date())
+    text = text + ' to '
+    if args.enddate is None:
+        text = text + str(datetime.now().date())
+    else:
+        text = text + str(args.enddate.date())
+
+    print '\n========================================================'
+    print text
+    print 'During the studied pediod, ' + str(TotalPeriodTags) + ' events out of ' + str(TotalPeriodEvents) + ' contains at least one tag with ' + args.tag + '.'
+    if TotalTags != 0:
+        print 'It represents ' + str(round(100*TotalPeriodTags/TotalTags, 3)) + '% of the fetched events (' + str(TotalTags) + ') including this tag.'
+    if TotalEvents != 0:
+        print 'It also represents ' + str(round(100*TotalPeriodTags/TotalEvents, 3)) + '% of all the fetched events (' + str(TotalEvents) + ').'
+

examples/situational-awareness/tags_count.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from pymisp import PyMISP
+from keys import misp_url, misp_key, misp_verifycert
+from datetime import datetime
+import argparse
+import json
+import tools
+
+def init(url, key):
+    return PyMISP(url, key, misp_verifycert, 'json')
+
+########## fetch data ##########
+
+def searchall(m, search, url):
+    result = m.search_all(search)
+    with open('data', 'w') as f:
+        f.write(json.dumps(result))
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py of searchall.py) and create a treemap epresenting the distribution of attributes in this sample.')
+    parser.add_argument("-s", "--search", help="string to search")
+    parser.add_argument("-b", "--begindate", help="The research will look for Tags attached to events posted at or after the given startdate (format: yyyy-mm-dd): If no date is given, default time is epoch time (1970-1-1)")
+    parser.add_argument("-e", "--enddate", help="The research will look for Tags attached to events posted at or before the given enddate (format: yyyy-mm-dd): If no date is given, default time is now()")
+
+    args = parser.parse_args()
+
+    misp = init(misp_url, misp_key)
+
+    if args.search is None:
+        args.search = ''
+    searchall(misp, args.search, misp_url)
+
+    if args.begindate is not None:
+        args.begindate = tools.toDatetime(args.begindate)
+    if args.enddate is not None:
+        args.enddate = tools.toDatetime(args.enddate)
+
+    Events = tools.eventsListBuildFromArray('data')
+    TotalEvents = tools.getNbitems(Events)
+    Tags = tools.tagsListBuild(Events)
+    result = tools.getNbOccurenceTags(Tags)
+    TotalTags = tools.getNbitems(Tags)
+
+    Events = tools.selectInRange(Events, begin=args.begindate, end=args.enddate)
+    TotalPeriodEvents = tools.getNbitems(Events)
+    Tags = tools.tagsListBuild(Events)
+    result = tools.getNbOccurenceTags(Tags)
+    TotalPeriodTags = tools.getNbitems(Tags)
+
+    text = 'Studied pediod: from '
+    if args.begindate is None:
+        text = text + '1970-01-01'
+    else:
+        text = text + str(args.begindate.date())
+    text = text + ' to '
+    if args.enddate is None:
+        text = text + str(datetime.now().date())
+    else:
+        text = text + str(args.enddate.date())
+
+    print '\n========================================================'
+    print text
+    print result
+    '''
+    print 'During the studied pediod, ' + str(TotalPeriodTags) + ' events out of ' + str(TotalPeriodEvents) + ' contains at least one tag with ' + args.tag + '.'
+    print 'It represents ' + str(round(100*TotalPeriodTags/TotalTags,3)) + '% of the fetched events (' + str(TotalTags) + ') including this tag.'
+    print 'It also represents ' + str(round(100*TotalPeriodTags/TotalEvents,3)) + '% of all the fetched events (' + str(TotalEvents) + ').'
+    '''

examples/situational-awareness/test_attribute_treemap.html

@@ -18,5 +18,9 @@
 		<td><iframe id="stats" src="attribute_table.html" frameBorder="0"></iframe></td> 
 		<td id="treemap"><object type="image/svg+xml" data="attribute_treemap.svg"></object></td>
 		</tr></table>
+	<!--
+		<div id="stats"><iframe src="table.html"></iframe></div> 
+		<div id="treemap"><object type="image/svg+xml" data="test.svg"></object></div>
+	-->
 	</body>
 </html>

examples/situational-awareness/tools.py

@@ -7,6 +7,52 @@ import random
 import pygal
 from pygal.style import Style
 import pandas as pd
+from datetime import datetime
+from datetime import timedelta
+from dateutil.parser import parse
+
+################ Tools ################
+
+def buildDoubleIndex(index1, index2, datatype):
+    it = -1
+    newindex1 = []
+    for index in index2:
+        if index == 0:
+            it+=1
+        newindex1.append(index1[it])
+    arrays =  [newindex1, index2]
+    tuples = list(zip(*arrays))
+    return pd.MultiIndex.from_tuples(tuples, names=['event', datatype])
+
+def buildNewColumn(index2, column):
+    it = -1
+    newcolumn = []
+    for index in index2:
+        if index == 0:
+            it+=1
+        newcolumn.append(column[it])
+    return newcolumn
+
+def dateInRange(datetimeTested, begin=None, end=None):
+    if begin == None:
+        begin = datetime(1970,1,1)
+    if end == None:
+        end = datetime.now()
+    return begin <= datetimeTested <= end
+
+def addColumn(dataframe, columnList, columnName):
+        dataframe.loc[:, columnName] = pd.Series(columnList, index=dataframe.index)
+
+def dateInRange(datetimeTested, begin=None, end=None):
+    if begin == None:
+        begin = datetime(1970,1,1)
+    if end == None:
+        end = datetime.now()
+    return begin <= datetimeTested <= end
+
+def toDatetime(date):
+    temp = date.split('-')
+    return datetime(int(temp[0]), int(temp[1]), int(temp[2]))
 
 ################ Formatting  ################
 
@@ -59,12 +105,58 @@ def attributesListBuild(Events):
         Attributes.append(pd.DataFrame(Attribute))
     return pd.concat(Attributes)
 
+def tagsListBuild(Events):
+    Tags = []
+    for Tag in Events['Tag']:
+        if type(Tag) is not list:
+            continue
+        Tags.append(pd.DataFrame(Tag))
+    Tags = pd.concat(Tags)
+    columnDate = buildNewColumn(Tags.index, Events['date'])
+    addColumn(Tags, columnDate, 'date')
+    index = buildDoubleIndex(Events.index, Tags.index, 'tag')
+    Tags = Tags.set_index(index)
+    return Tags
+
+def selectInRange(Events, begin=None, end=None):
+    inRange = []
+    for i, Event in Events.iterrows():
+        if dateInRange(parse(Event['date']), begin, end):
+            inRange.append(Event.tolist())
+    inRange = pd.DataFrame(inRange)
+    temp = Events.columns.tolist()
+    inRange.columns = temp
+    return inRange
+'''
+def isTagIn(dataframe, tag):
+    print 'tag =' + tag
+    result = []
+    for tagname in dataframe['name']:
+        print tagname
+        if tag in tagname:
+            print 'True'
+            result.append(tagname)
+    return result
+'''
+
+def isTagIn(dataframe, tag):
+    temp = Tags[Tags['name'].str.contains(test)].index.tolist()
+    index = []
+    for i in range(len(temp)):
+        if temp[i][0] not in index:
+            index.append(temp[i][0])
+    return index
 
 ################ Basic Stats ################
 
+def getNbitems(dataframe):
+        return len(dataframe.index)
+
 def getNbAttributePerEventCategoryType(Attributes):
     return Attributes.groupby(['event_id', 'category', 'type']).count()['id']
 
+def getNbOccurenceTags(Tags):
+        return Tags.groupby('name').count()['id']
 
 ################ Charts ################
 

pymisp/__init__.py

@@ -1,3 +1,3 @@
-__version__ = '2.4.48.1'
+__version__ = '2.4.48.2'
 
 from .api import PyMISP, PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey

pymisp/api.py

@@ -364,6 +364,13 @@ class PyMISP(object):
 
         return self._check_response(response)
 
+    def remove_tag(self, event, tag):
+        session = self.__prepare_session('json')
+        to_post = {'request': {'Event': {'id': event['Event']['id'], 'tag': tag}}}
+        response = session.post(urljoin(self.root_url, 'events/removeTag'), data=json.dumps(to_post))
+
+        return self._check_response(response)
+
     def change_threat_level(self, event, threat_level_id):
         event['Event']['threat_level_id'] = threat_level_id
         self._prepare_update(event)