diff --git a/examples/situational-awareness/README.md b/examples/situational-awareness/README.md index f0e4b19..58a1381 100644 --- a/examples/situational-awareness/README.md +++ b/examples/situational-awareness/README.md @@ -4,6 +4,15 @@ * It will also generate a html document with a table (attribute\_table.html) containing count for each type of attribute. * test\_attribute\_treemap.html is a quick page made to visualize both treemap and table at the same time. +* tags\_count.py is a script that count the number of occurences of every tags in a fetched sample of Events in a given period of time. +* tag\_search.py is a script that count the number of occurences of a given tag in a fetched sample of Events in a given period of time. + * Events will be fetched from _days_ days ago to today. + * _begindate_ is the beginning of the studied period. If it is later than today, an error will be raised. + * _enddate_ is the end of the studied period. If it is earlier than _begindate_, an error will be raised. + * tag\_search.py allows research for multiple tags is possible by separating each tag by the | symbol. + * Partial research is also possible with tag\_search.py. For instance, search for "ransom" will also return tags containin "ransomware". +:warning: These scripts are not time optimised + ## Requierements * [Pygal](https://github.com/Kozea/pygal/) diff --git a/examples/situational-awareness/tag_search.py b/examples/situational-awareness/tag_search.py index e862ce1..d695a00 100644 --- a/examples/situational-awareness/tag_search.py +++ b/examples/situational-awareness/tag_search.py @@ -21,7 +21,7 @@ def download_last(m, last): if __name__ == '__main__': parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py) and give the number of occurrence of the given tag in this sample.') parser.add_argument("-t", "--tag", required=True, help="tag to search (search for multiple tags is possible by using |. example : \"osint|OSINT\")") - parser.add_argument("-d", "--days", help="number of days before today to search. If not define, default value is 7") + parser.add_argument("-d", "--days", type=int, help="number of days before today to search. If not define, default value is 7") parser.add_argument("-b", "--begindate", help="The research will look for tags attached to events posted at or after the given startdate (format: yyyy-mm-dd): If no date is given, default time is epoch time (1970-1-1)") parser.add_argument("-e", "--enddate", help="The research will look for tags attached to events posted at or before the given enddate (format: yyyy-mm-dd): If no date is given, default time is now()") @@ -30,21 +30,22 @@ if __name__ == '__main__': misp = init(misp_url, misp_key) if args.days is None: - args.days = '7' - download_last(misp, args.days + 'd') + args.days = 7 + download_last(misp, str(args.days) + 'd') - if args.begindate is not None: - args.begindate = tools.toDatetime(args.begindate) - if args.enddate is not None: - args.enddate = tools.toDatetime(args.enddate) + tools.checkDateConsistancy(args.begindate, args.enddate, tools.getLastdate(args.days)) - Events = tools.eventsListBuildFromArray('data') - TotalEvents = tools.getNbitems(Events) - Tags = tools.tagsListBuild(Events) - result = tools.isTagIn(Tags, args.tag) - TotalTags = len(result) + if args.begindate is None: + args.begindate = tools.getLastdate(args.days) + else: + args.begindate = tools.setBegindate(tools.toDatetime(args.begindate), tools.getLastdate(args.days)) - Events = tools.selectInRange(Events, begin=args.begindate, end=args.enddate) + if args.enddate is None: + args.enddate = datetime.now() + else: + args.enddate = tools.setEnddate(tools.toDatetime(args.enddate)) + + Events = tools.selectInRange(tools.eventsListBuildFromArray('data'), begin=args.begindate, end=args.enddate) TotalPeriodEvents = tools.getNbitems(Events) Tags = tools.tagsListBuild(Events) result = tools.isTagIn(Tags, args.tag) @@ -64,8 +65,5 @@ if __name__ == '__main__': print '\n========================================================' print text print 'During the studied pediod, ' + str(TotalPeriodTags) + ' events out of ' + str(TotalPeriodEvents) + ' contains at least one tag with ' + args.tag + '.' - if TotalTags != 0: - print 'It represents ' + str(round(100*TotalPeriodTags/TotalTags, 3)) + '% of the fetched events (' + str(TotalTags) + ') including this tag.' - if TotalEvents != 0: - print 'It also represents ' + str(round(100*TotalPeriodTags/TotalEvents, 3)) + '% of all the fetched events (' + str(TotalEvents) + ').' - + if TotalPeriodEvents != 0: + print 'It represents ' + str(round(100*TotalPeriodTags/TotalPeriodEvents, 3)) + '% of the events in this period.' diff --git a/examples/situational-awareness/tags_count.py b/examples/situational-awareness/tags_count.py index 80dfdcd..58e6194 100644 --- a/examples/situational-awareness/tags_count.py +++ b/examples/situational-awareness/tags_count.py @@ -20,24 +20,29 @@ def download_last(m, last): if __name__ == '__main__': parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py) and give the repartition of tags in this sample.') - parser.add_argument("-d", "--days", help="number of days before today to search. If not define, default value is 7") + parser.add_argument("-d", "--days", type=int, help="number of days before today to search. If not define, default value is 7") parser.add_argument("-b", "--begindate", help="The research will look for tags attached to events posted at or after the given startdate (format: yyyy-mm-dd): If no date is given, default time is epoch time (1970-1-1)") parser.add_argument("-e", "--enddate", help="The research will look for tags attached to events posted at or before the given enddate (format: yyyy-mm-dd): If no date is given, default time is now()") - - args = parser.parse_args() misp = init(misp_url, misp_key) if args.days is None: - args.days = '7' - download_last(misp, args.days + 'd') + args.days = 7 + download_last(misp, str(args.days) + 'd') - if args.begindate is not None: - args.begindate = tools.toDatetime(args.begindate) - if args.enddate is not None: - args.enddate = tools.toDatetime(args.enddate) + tools.checkDateConsistancy(args.begindate, args.enddate, tools.getLastdate(args.days)) + + if args.begindate is None: + args.begindate = tools.getLastdate(args.days) + else: + args.begindate = tools.setBegindate(tools.toDatetime(args.begindate), tools.getLastdate(args.days)) + + if args.enddate is None: + args.enddate = datetime.now() + else: + args.enddate = tools.setEnddate(tools.toDatetime(args.enddate)) Events = tools.eventsListBuildFromArray('data') TotalEvents = tools.getNbitems(Events) diff --git a/examples/situational-awareness/tools.py b/examples/situational-awareness/tools.py index 57ee710..098d035 100644 --- a/examples/situational-awareness/tools.py +++ b/examples/situational-awareness/tools.py @@ -11,6 +11,15 @@ import pandas as pd from datetime import datetime from datetime import timedelta from dateutil.parser import parse +import sys + +################ Errors ################ + +class DateError(Exception): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) ################ Tools ################ @@ -52,8 +61,41 @@ def dateInRange(datetimeTested, begin=None, end=None): return begin <= datetimeTested <= end def toDatetime(date): - temp = date.split('-') - return datetime(int(temp[0]), int(temp[1]), int(temp[2])) + return parse(date) + +def checkDateConsistancy(begindate, enddate, lastdate): + try: + if begindate is not None and enddate is not None: + if begindate > enddate: + raise DateError('begindate (' + begindate + ') cannot be after enddate (' + enddate + ')') + except DateError as e: + print('DateError: ' + e.value) + sys.exit(1) + + try: + if enddate is not None: + if toDatetime(enddate) < lastdate: + raise DateError('enddate (' + enddate + ') cannot be before lastdate (' + str(lastdate) + ')' ) + except DateError as e: + print('DateError: ' + e.value) + sys.exit(1) + + try: + if begindate is not None: + if toDatetime(begindate) > datetime.now(): + raise DateError('begindate (' + begindate + ') cannot be after today (' + str(datetime.now().date()) + ')') + except DateError as e: + print('DateError: ' + e.value) + sys.exit(1) + +def setBegindate(begindate, lastdate): + return max(begindate, lastdate) + +def setEnddate(enddate): + return min(enddate, datetime.now()) + +def getLastdate(last): + return (datetime.now() - timedelta(days=int(last))).replace(hour=0, minute=0, second=0, microsecond=0) ################ Formatting ################