Make printed date more consistent + update README.md

examples/situational-awareness/README.md

@@ -4,6 +4,15 @@
 * It will also generate a html document with a table (attribute\_table.html) containing count for each type of attribute.
 * test\_attribute\_treemap.html is a quick page made to visualize both treemap and table at the same time.
 
+* tags\_count.py is a script that count the number of occurences of every tags in a fetched sample of Events in a given period of time.
+* tag\_search.py is a script that count the number of occurences of a given tag  in a fetched sample of Events in a given period of time.
+    * Events will be fetched from _days_ days ago to today.
+    * _begindate_ is the beginning of the studied period. If it is later than today, an error will be raised.
+    * _enddate_ is the end of the studied period. If it is earlier than _begindate_, an error will be raised.
+    * tag\_search.py allows research for multiple tags is possible by separating each tag by the | symbol.
+    * Partial research is also possible with tag\_search.py. For instance, search for "ransom" will also return tags containin "ransomware".
+:warning: These scripts are not time optimised
+
 ## Requierements
 
 * [Pygal](https://github.com/Kozea/pygal/)

examples/situational-awareness/tag_search.py

@@ -21,7 +21,7 @@ def download_last(m, last):
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py) and give the number of occurrence of the given tag in this sample.')
     parser.add_argument("-t", "--tag", required=True, help="tag to search (search for multiple tags is possible by using |. example : \"osint|OSINT\")")
-    parser.add_argument("-d", "--days", help="number of days before today to search. If not define, default value is 7")
+    parser.add_argument("-d", "--days", type=int, help="number of days before today to search. If not define, default value is 7")
     parser.add_argument("-b", "--begindate", help="The research will look for tags attached to events posted at or after the given startdate (format: yyyy-mm-dd): If no date is given, default time is epoch time (1970-1-1)")
     parser.add_argument("-e", "--enddate", help="The research will look for tags attached to events posted at or before the given enddate (format: yyyy-mm-dd): If no date is given, default time is now()")
 
@@ -30,21 +30,22 @@ if __name__ == '__main__':
     misp = init(misp_url, misp_key)
 
     if args.days is None:
-        args.days = '7'
+        args.days = 7
-    download_last(misp, args.days + 'd')
+    download_last(misp, str(args.days) + 'd')
 
-    if args.begindate is not None:
+    tools.checkDateConsistancy(args.begindate, args.enddate, tools.getLastdate(args.days))
-        args.begindate = tools.toDatetime(args.begindate)
-    if args.enddate is not None:
-        args.enddate = tools.toDatetime(args.enddate)
 
-    Events = tools.eventsListBuildFromArray('data')
+    if args.begindate is None:
-    TotalEvents = tools.getNbitems(Events)
+        args.begindate = tools.getLastdate(args.days)
-    Tags = tools.tagsListBuild(Events)
+    else:
-    result = tools.isTagIn(Tags, args.tag)
+        args.begindate = tools.setBegindate(tools.toDatetime(args.begindate), tools.getLastdate(args.days))
-    TotalTags = len(result)
 
-    Events = tools.selectInRange(Events, begin=args.begindate, end=args.enddate)
+    if args.enddate is None:
+        args.enddate = datetime.now()
+    else:
+        args.enddate = tools.setEnddate(tools.toDatetime(args.enddate))
+
+    Events = tools.selectInRange(tools.eventsListBuildFromArray('data'), begin=args.begindate, end=args.enddate)
     TotalPeriodEvents = tools.getNbitems(Events)
     Tags = tools.tagsListBuild(Events)
     result = tools.isTagIn(Tags, args.tag)
@@ -64,8 +65,5 @@ if __name__ == '__main__':
     print '\n========================================================'
     print text
     print 'During the studied pediod, ' + str(TotalPeriodTags) + ' events out of ' + str(TotalPeriodEvents) + ' contains at least one tag with ' + args.tag + '.'
-    if TotalTags != 0:
+    if TotalPeriodEvents != 0:
-        print 'It represents ' + str(round(100*TotalPeriodTags/TotalTags, 3)) + '% of the fetched events (' + str(TotalTags) + ') including this tag.'
+        print 'It represents ' + str(round(100*TotalPeriodTags/TotalPeriodEvents, 3)) + '% of the events in this period.'
-    if TotalEvents != 0:
-        print 'It also represents ' + str(round(100*TotalPeriodTags/TotalEvents, 3)) + '% of all the fetched events (' + str(TotalEvents) + ').'
-

examples/situational-awareness/tags_count.py

@@ -20,24 +20,29 @@ def download_last(m, last):
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py) and give the repartition of tags in this sample.')
-    parser.add_argument("-d", "--days", help="number of days before today to search. If not define, default value is 7")
+    parser.add_argument("-d", "--days", type=int, help="number of days before today to search. If not define, default value is 7")
     parser.add_argument("-b", "--begindate", help="The research will look for tags attached to events posted at or after the given startdate (format: yyyy-mm-dd): If no date is given, default time is epoch time (1970-1-1)")
     parser.add_argument("-e", "--enddate", help="The research will look for tags attached to events posted at or before the given enddate (format: yyyy-mm-dd): If no date is given, default time is now()")
 
-
-
     args = parser.parse_args()
 
     misp = init(misp_url, misp_key)
 
     if args.days is None:
-        args.days = '7'
+        args.days = 7
-    download_last(misp, args.days + 'd')
+    download_last(misp, str(args.days) + 'd')
 
-    if args.begindate is not None:
+    tools.checkDateConsistancy(args.begindate, args.enddate, tools.getLastdate(args.days))
-        args.begindate = tools.toDatetime(args.begindate)
+
-    if args.enddate is not None:
+    if args.begindate is None:
-        args.enddate = tools.toDatetime(args.enddate)
+        args.begindate = tools.getLastdate(args.days)
+    else:
+        args.begindate = tools.setBegindate(tools.toDatetime(args.begindate), tools.getLastdate(args.days))
+
+    if args.enddate is None:
+        args.enddate = datetime.now()
+    else:
+        args.enddate = tools.setEnddate(tools.toDatetime(args.enddate))
 
     Events = tools.eventsListBuildFromArray('data')
     TotalEvents = tools.getNbitems(Events)

examples/situational-awareness/tools.py

@@ -11,6 +11,15 @@ import pandas as pd
 from datetime import datetime
 from datetime import timedelta
 from dateutil.parser import parse
+import sys
+
+################ Errors ################
+
+class DateError(Exception):
+    def __init__(self, value):
+        self.value = value
+    def __str__(self):
+        return repr(self.value)
 
 ################ Tools ################
 
@@ -52,8 +61,41 @@ def dateInRange(datetimeTested, begin=None, end=None):
     return begin <= datetimeTested <= end
 
 def toDatetime(date):
-    temp = date.split('-')
+    return parse(date)
-    return datetime(int(temp[0]), int(temp[1]), int(temp[2]))
+
+def checkDateConsistancy(begindate, enddate, lastdate):
+    try:
+        if begindate is not None and enddate is not None:
+            if begindate > enddate:
+                raise DateError('begindate (' + begindate +  ') cannot be after enddate (' + enddate + ')')
+    except DateError as e:
+        print('DateError: ' + e.value)
+        sys.exit(1)
+
+    try:
+        if enddate is not None:
+            if toDatetime(enddate) < lastdate:
+                raise DateError('enddate (' + enddate + ') cannot be before lastdate (' + str(lastdate) + ')' )
+    except DateError as e:
+        print('DateError: ' + e.value)
+        sys.exit(1)
+
+    try:
+        if begindate is not None:
+            if toDatetime(begindate) > datetime.now():
+                raise DateError('begindate (' + begindate + ') cannot be after today (' + str(datetime.now().date()) + ')')
+    except DateError as e:
+        print('DateError: ' + e.value)
+        sys.exit(1)
+
+def setBegindate(begindate, lastdate):
+    return max(begindate, lastdate)
+
+def setEnddate(enddate):
+    return min(enddate, datetime.now())
+
+def getLastdate(last):
+    return (datetime.now() - timedelta(days=int(last))).replace(hour=0, minute=0, second=0, microsecond=0)
 
 ################ Formatting  ################