diff --git a/examples/situational-awareness/README.md b/examples/situational-awareness/README.md new file mode 100644 index 0000000..86f06bd --- /dev/null +++ b/examples/situational-awareness/README.md @@ -0,0 +1,19 @@ +## Explanation + +* treemap.py is a script that will generate an interactive svg (attribute\_treemap.svg) containing a treepmap representing the distribution of attributes in a sample (data) fetched from the instance using "last" or "searchall" examples. +* It will also generate a html document with a table (attribute\_table.html) containing count for each type of attribute. +* test\_attribute\_treemap.html is a quick page made to visualize both treemap and table at the same time. + +* tags\_count.py is a script that count the number of occurences of every tags in a fetched sample of Events in a given period of time. +* tag\_search.py is a script that count the number of occurences of a given tag in a fetched sample of Events in a given period of time. + * Events will be fetched from _days_ days ago to today. + * _begindate_ is the beginning of the studied period. If it is later than today, an error will be raised. + * _enddate_ is the end of the studied period. If it is earlier than _begindate_, an error will be raised. + * tag\_search.py allows research for multiple tags is possible by separating each tag by the | symbol. + * Partial research is also possible with tag\_search.py. For instance, search for "ransom" will also return tags containin "ransomware". + +:warning: These scripts are not time optimised + +## Requierements + +* [Pygal](https://github.com/Kozea/pygal/) diff --git a/examples/treemap/treemap.py b/examples/situational-awareness/attribute_treemap.py similarity index 100% rename from examples/treemap/treemap.py rename to examples/situational-awareness/attribute_treemap.py diff --git a/examples/treemap/style.css b/examples/situational-awareness/style.css similarity index 84% rename from examples/treemap/style.css rename to examples/situational-awareness/style.css index 03f1ec9..8c5313b 100644 --- a/examples/treemap/style.css +++ b/examples/situational-awareness/style.css @@ -1,5 +1,6 @@ body { + /*font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;*/ font-family: Consolas, "Liberation Mono", Menlo, Courier, monospace; } @@ -10,6 +11,8 @@ h1 text-align:center; } +/*** Stats Tables ***/ + table { border-collapse: collapse; diff --git a/examples/situational-awareness/tag_search.py b/examples/situational-awareness/tag_search.py new file mode 100644 index 0000000..d695a00 --- /dev/null +++ b/examples/situational-awareness/tag_search.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pymisp import PyMISP +from keys import misp_url, misp_key, misp_verifycert +from datetime import datetime +import argparse +import json +import tools + +def init(url, key): + return PyMISP(url, key, misp_verifycert, 'json') + +########## fetch data ########## + +def download_last(m, last): + result = m.download_last(last) + with open('data', 'w') as f: + f.write(json.dumps(result)) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py) and give the number of occurrence of the given tag in this sample.') + parser.add_argument("-t", "--tag", required=True, help="tag to search (search for multiple tags is possible by using |. example : \"osint|OSINT\")") + parser.add_argument("-d", "--days", type=int, help="number of days before today to search. If not define, default value is 7") + parser.add_argument("-b", "--begindate", help="The research will look for tags attached to events posted at or after the given startdate (format: yyyy-mm-dd): If no date is given, default time is epoch time (1970-1-1)") + parser.add_argument("-e", "--enddate", help="The research will look for tags attached to events posted at or before the given enddate (format: yyyy-mm-dd): If no date is given, default time is now()") + + args = parser.parse_args() + + misp = init(misp_url, misp_key) + + if args.days is None: + args.days = 7 + download_last(misp, str(args.days) + 'd') + + tools.checkDateConsistancy(args.begindate, args.enddate, tools.getLastdate(args.days)) + + if args.begindate is None: + args.begindate = tools.getLastdate(args.days) + else: + args.begindate = tools.setBegindate(tools.toDatetime(args.begindate), tools.getLastdate(args.days)) + + if args.enddate is None: + args.enddate = datetime.now() + else: + args.enddate = tools.setEnddate(tools.toDatetime(args.enddate)) + + Events = tools.selectInRange(tools.eventsListBuildFromArray('data'), begin=args.begindate, end=args.enddate) + TotalPeriodEvents = tools.getNbitems(Events) + Tags = tools.tagsListBuild(Events) + result = tools.isTagIn(Tags, args.tag) + TotalPeriodTags = len(result) + + text = 'Studied pediod: from ' + if args.begindate is None: + text = text + '1970-01-01' + else: + text = text + str(args.begindate.date()) + text = text + ' to ' + if args.enddate is None: + text = text + str(datetime.now().date()) + else: + text = text + str(args.enddate.date()) + + print '\n========================================================' + print text + print 'During the studied pediod, ' + str(TotalPeriodTags) + ' events out of ' + str(TotalPeriodEvents) + ' contains at least one tag with ' + args.tag + '.' + if TotalPeriodEvents != 0: + print 'It represents ' + str(round(100*TotalPeriodTags/TotalPeriodEvents, 3)) + '% of the events in this period.' diff --git a/examples/situational-awareness/tags_count.py b/examples/situational-awareness/tags_count.py new file mode 100644 index 0000000..58e6194 --- /dev/null +++ b/examples/situational-awareness/tags_count.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pymisp import PyMISP +from keys import misp_url, misp_key, misp_verifycert +from datetime import datetime +import argparse +import json +import tools + +def init(url, key): + return PyMISP(url, key, misp_verifycert, 'json') + +########## fetch data ########## + +def download_last(m, last): + result = m.download_last(last) + with open('data', 'w') as f: + f.write(json.dumps(result)) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py) and give the repartition of tags in this sample.') + parser.add_argument("-d", "--days", type=int, help="number of days before today to search. If not define, default value is 7") + parser.add_argument("-b", "--begindate", help="The research will look for tags attached to events posted at or after the given startdate (format: yyyy-mm-dd): If no date is given, default time is epoch time (1970-1-1)") + parser.add_argument("-e", "--enddate", help="The research will look for tags attached to events posted at or before the given enddate (format: yyyy-mm-dd): If no date is given, default time is now()") + + args = parser.parse_args() + + misp = init(misp_url, misp_key) + + if args.days is None: + args.days = 7 + download_last(misp, str(args.days) + 'd') + + tools.checkDateConsistancy(args.begindate, args.enddate, tools.getLastdate(args.days)) + + if args.begindate is None: + args.begindate = tools.getLastdate(args.days) + else: + args.begindate = tools.setBegindate(tools.toDatetime(args.begindate), tools.getLastdate(args.days)) + + if args.enddate is None: + args.enddate = datetime.now() + else: + args.enddate = tools.setEnddate(tools.toDatetime(args.enddate)) + + Events = tools.eventsListBuildFromArray('data') + TotalEvents = tools.getNbitems(Events) + Tags = tools.tagsListBuild(Events) + result = tools.getNbOccurenceTags(Tags) + TotalTags = tools.getNbitems(Tags) + + Events = tools.selectInRange(Events, begin=args.begindate, end=args.enddate) + TotalPeriodEvents = tools.getNbitems(Events) + Tags = tools.tagsListBuild(Events) + result = tools.getNbOccurenceTags(Tags) + TotalPeriodTags = tools.getNbitems(Tags) + + text = 'Studied pediod: from ' + if args.begindate is None: + text = text + '1970-01-01' + else: + text = text + str(args.begindate.date()) + text = text + ' to ' + if args.enddate is None: + text = text + str(datetime.now().date()) + else: + text = text + str(args.enddate.date()) + + print '\n========================================================' + print text + print result diff --git a/examples/treemap/test_attribute_treemap.html b/examples/situational-awareness/test_attribute_treemap.html similarity index 70% rename from examples/treemap/test_attribute_treemap.html rename to examples/situational-awareness/test_attribute_treemap.html index 8d6dd1b..d6e8fc4 100644 --- a/examples/treemap/test_attribute_treemap.html +++ b/examples/situational-awareness/test_attribute_treemap.html @@ -18,5 +18,9 @@