From b0a66da4deca32d0fb677d8ad38623b63902a319 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Thu, 21 Jul 2016 10:06:47 +0200 Subject: [PATCH 1/2] handling some NaN exceptions --- examples/situational-awareness/tools.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/examples/situational-awareness/tools.py b/examples/situational-awareness/tools.py index f259f9c..57ee710 100644 --- a/examples/situational-awareness/tools.py +++ b/examples/situational-awareness/tools.py @@ -3,6 +3,7 @@ import json from json import JSONDecoder +import math import random import pygal from pygal.style import Style @@ -57,7 +58,7 @@ def toDatetime(date): ################ Formatting ################ def eventsListBuildFromList(filename): - with open('testt', 'r') as myfile: + with open(filename, 'r') as myfile: s=myfile.read().replace('\n', '') decoder = JSONDecoder() s_len = len(s) @@ -92,7 +93,7 @@ def eventsListBuildFromArray(filename): data.append(pd.DataFrame.from_dict(e, orient='index')) Events = pd.concat(data) for it in range(Events['attribute_count'].size): - if Events['attribute_count'][it] == None: + if Events['attribute_count'][it] == None or (isinstance(Events['attribute_count'][it], float) and math.isnan(Events['attribute_count'][it])): Events['attribute_count'][it]='0' else: Events['attribute_count'][it]=int(Events['attribute_count'][it]) @@ -127,20 +128,9 @@ def selectInRange(Events, begin=None, end=None): temp = Events.columns.tolist() inRange.columns = temp return inRange -''' -def isTagIn(dataframe, tag): - print 'tag =' + tag - result = [] - for tagname in dataframe['name']: - print tagname - if tag in tagname: - print 'True' - result.append(tagname) - return result -''' def isTagIn(dataframe, tag): - temp = Tags[Tags['name'].str.contains(test)].index.tolist() + temp = dataframe[dataframe['name'].str.contains(tag)].index.tolist() index = [] for i in range(len(temp)): if temp[i][0] not in index: From 0f68ffc61755b3b5c1ee558b535d2a3c4d78ca86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9borah=20Servili?= Date: Thu, 21 Jul 2016 10:09:10 +0200 Subject: [PATCH 2/2] modify fetching method to use last --- examples/situational-awareness/tag_search.py | 18 +++++++------- examples/situational-awareness/tags_count.py | 25 +++++++++----------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/examples/situational-awareness/tag_search.py b/examples/situational-awareness/tag_search.py index a04f54a..e862ce1 100644 --- a/examples/situational-awareness/tag_search.py +++ b/examples/situational-awareness/tag_search.py @@ -13,23 +13,25 @@ def init(url, key): ########## fetch data ########## -def searchall(m, search, url): - result = m.search_all(search) +def download_last(m, last): + result = m.download_last(last) with open('data', 'w') as f: f.write(json.dumps(result)) if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py of searchall.py) and create a treemap epresenting the distribution of attributes in this sample.') - parser.add_argument("-s", "--search", help="string to search") - parser.add_argument("-t", "--tag", required=True, help="String to search in tags, can be composed. Example: \"ransomware|Ransomware\"") - parser.add_argument("-b", "--begindate", help="The research will look for Tags attached to events posted at or after the given startdate (format: yyyy-mm-dd): If no date is given, default time is epoch time (1970-1-1)") - parser.add_argument("-e", "--enddate", help="The research will look for Tags attached to events posted at or before the given enddate (format: yyyy-mm-dd): If no date is given, default time is now()") + parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py) and give the number of occurrence of the given tag in this sample.') + parser.add_argument("-t", "--tag", required=True, help="tag to search (search for multiple tags is possible by using |. example : \"osint|OSINT\")") + parser.add_argument("-d", "--days", help="number of days before today to search. If not define, default value is 7") + parser.add_argument("-b", "--begindate", help="The research will look for tags attached to events posted at or after the given startdate (format: yyyy-mm-dd): If no date is given, default time is epoch time (1970-1-1)") + parser.add_argument("-e", "--enddate", help="The research will look for tags attached to events posted at or before the given enddate (format: yyyy-mm-dd): If no date is given, default time is now()") args = parser.parse_args() misp = init(misp_url, misp_key) - searchall(misp, args.search, misp_url) + if args.days is None: + args.days = '7' + download_last(misp, args.days + 'd') if args.begindate is not None: args.begindate = tools.toDatetime(args.begindate) diff --git a/examples/situational-awareness/tags_count.py b/examples/situational-awareness/tags_count.py index cff5d9b..80dfdcd 100644 --- a/examples/situational-awareness/tags_count.py +++ b/examples/situational-awareness/tags_count.py @@ -13,24 +13,26 @@ def init(url, key): ########## fetch data ########## -def searchall(m, search, url): - result = m.search_all(search) +def download_last(m, last): + result = m.download_last(last) with open('data', 'w') as f: f.write(json.dumps(result)) if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py of searchall.py) and create a treemap epresenting the distribution of attributes in this sample.') - parser.add_argument("-s", "--search", help="string to search") - parser.add_argument("-b", "--begindate", help="The research will look for Tags attached to events posted at or after the given startdate (format: yyyy-mm-dd): If no date is given, default time is epoch time (1970-1-1)") - parser.add_argument("-e", "--enddate", help="The research will look for Tags attached to events posted at or before the given enddate (format: yyyy-mm-dd): If no date is given, default time is now()") + parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py) and give the repartition of tags in this sample.') + parser.add_argument("-d", "--days", help="number of days before today to search. If not define, default value is 7") + parser.add_argument("-b", "--begindate", help="The research will look for tags attached to events posted at or after the given startdate (format: yyyy-mm-dd): If no date is given, default time is epoch time (1970-1-1)") + parser.add_argument("-e", "--enddate", help="The research will look for tags attached to events posted at or before the given enddate (format: yyyy-mm-dd): If no date is given, default time is now()") + + args = parser.parse_args() misp = init(misp_url, misp_key) - if args.search is None: - args.search = '' - searchall(misp, args.search, misp_url) + if args.days is None: + args.days = '7' + download_last(misp, args.days + 'd') if args.begindate is not None: args.begindate = tools.toDatetime(args.begindate) @@ -63,8 +65,3 @@ if __name__ == '__main__': print '\n========================================================' print text print result - ''' - print 'During the studied pediod, ' + str(TotalPeriodTags) + ' events out of ' + str(TotalPeriodEvents) + ' contains at least one tag with ' + args.tag + '.' - print 'It represents ' + str(round(100*TotalPeriodTags/TotalTags,3)) + '% of the fetched events (' + str(TotalTags) + ') including this tag.' - print 'It also represents ' + str(round(100*TotalPeriodTags/TotalEvents,3)) + '% of all the fetched events (' + str(TotalEvents) + ').' - '''