Merge pull request #3 from Delta-Sierra/newbranch

examples/situational-awareness/tag_search.py

@@ -13,23 +13,25 @@ def init(url, key):
 
 ########## fetch data ##########
 
-def searchall(m, search, url):
+def download_last(m, last):
-    result = m.search_all(search)
+    result = m.download_last(last)
     with open('data', 'w') as f:
         f.write(json.dumps(result))
 
 if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py of searchall.py) and create a treemap epresenting the distribution of attributes in this sample.')
+    parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py) and give the number of occurrence of the given tag in this sample.')
-    parser.add_argument("-s", "--search", help="string to search")
+    parser.add_argument("-t", "--tag", required=True, help="tag to search (search for multiple tags is possible by using |. example : \"osint|OSINT\")")
-    parser.add_argument("-t", "--tag", required=True, help="String to search in tags, can be composed. Example: \"ransomware|Ransomware\"")
+    parser.add_argument("-d", "--days", help="number of days before today to search. If not define, default value is 7")
-    parser.add_argument("-b", "--begindate", help="The research will look for Tags attached to events posted at or after the given startdate (format: yyyy-mm-dd): If no date is given, default time is epoch time (1970-1-1)")
+    parser.add_argument("-b", "--begindate", help="The research will look for tags attached to events posted at or after the given startdate (format: yyyy-mm-dd): If no date is given, default time is epoch time (1970-1-1)")
-    parser.add_argument("-e", "--enddate", help="The research will look for Tags attached to events posted at or before the given enddate (format: yyyy-mm-dd): If no date is given, default time is now()")
+    parser.add_argument("-e", "--enddate", help="The research will look for tags attached to events posted at or before the given enddate (format: yyyy-mm-dd): If no date is given, default time is now()")
 
     args = parser.parse_args()
 
     misp = init(misp_url, misp_key)
 
-    searchall(misp, args.search, misp_url)
+    if args.days is None:
+        args.days = '7'
+    download_last(misp, args.days + 'd')
 
     if args.begindate is not None:
         args.begindate = tools.toDatetime(args.begindate)

examples/situational-awareness/tags_count.py

@@ -13,24 +13,26 @@ def init(url, key):
 
 ########## fetch data ##########
 
-def searchall(m, search, url):
+def download_last(m, last):
-    result = m.search_all(search)
+    result = m.download_last(last)
     with open('data', 'w') as f:
         f.write(json.dumps(result))
 
 if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py of searchall.py) and create a treemap epresenting the distribution of attributes in this sample.')
+    parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py) and give the repartition of tags in this sample.')
-    parser.add_argument("-s", "--search", help="string to search")
+    parser.add_argument("-d", "--days", help="number of days before today to search. If not define, default value is 7")
-    parser.add_argument("-b", "--begindate", help="The research will look for Tags attached to events posted at or after the given startdate (format: yyyy-mm-dd): If no date is given, default time is epoch time (1970-1-1)")
+    parser.add_argument("-b", "--begindate", help="The research will look for tags attached to events posted at or after the given startdate (format: yyyy-mm-dd): If no date is given, default time is epoch time (1970-1-1)")
-    parser.add_argument("-e", "--enddate", help="The research will look for Tags attached to events posted at or before the given enddate (format: yyyy-mm-dd): If no date is given, default time is now()")
+    parser.add_argument("-e", "--enddate", help="The research will look for tags attached to events posted at or before the given enddate (format: yyyy-mm-dd): If no date is given, default time is now()")
+
+
 
     args = parser.parse_args()
 
     misp = init(misp_url, misp_key)
 
-    if args.search is None:
+    if args.days is None:
-        args.search = ''
+        args.days = '7'
-    searchall(misp, args.search, misp_url)
+    download_last(misp, args.days + 'd')
 
     if args.begindate is not None:
         args.begindate = tools.toDatetime(args.begindate)
@@ -63,8 +65,3 @@ if __name__ == '__main__':
     print '\n========================================================'
     print text
     print result
-    '''
-    print 'During the studied pediod, ' + str(TotalPeriodTags) + ' events out of ' + str(TotalPeriodEvents) + ' contains at least one tag with ' + args.tag + '.'
-    print 'It represents ' + str(round(100*TotalPeriodTags/TotalTags,3)) + '% of the fetched events (' + str(TotalTags) + ') including this tag.'
-    print 'It also represents ' + str(round(100*TotalPeriodTags/TotalEvents,3)) + '% of all the fetched events (' + str(TotalEvents) + ').'
-    '''

examples/situational-awareness/tools.py

@@ -3,6 +3,7 @@
 
 import json
 from json import JSONDecoder
+import math
 import random
 import pygal
 from pygal.style import Style
@@ -57,7 +58,7 @@ def toDatetime(date):
 ################ Formatting  ################
 
 def eventsListBuildFromList(filename):
-    with open('testt', 'r') as myfile:
+    with open(filename, 'r') as myfile:
         s=myfile.read().replace('\n', '')
     decoder = JSONDecoder()
     s_len = len(s)
@@ -92,7 +93,7 @@ def eventsListBuildFromArray(filename):
         data.append(pd.DataFrame.from_dict(e, orient='index'))
     Events = pd.concat(data)
     for it in range(Events['attribute_count'].size):
-        if Events['attribute_count'][it] == None:
+        if Events['attribute_count'][it] == None or (isinstance(Events['attribute_count'][it], float) and math.isnan(Events['attribute_count'][it])):
             Events['attribute_count'][it]='0'
         else:
             Events['attribute_count'][it]=int(Events['attribute_count'][it])