misp-dashboard/helpers/trendings_helper.py

207 lines
8.8 KiB
Python

import math, random
import os
import json
import copy
import datetime, time
import logging
from collections import OrderedDict
import util
class Trendings_helper:
def __init__(self, serv_redis_db, cfg):
self.serv_redis_db = serv_redis_db
self.cfg = cfg
# REDIS keys
self.keyEvent = "TRENDINGS_EVENTS"
self.keyCateg = "TRENDINGS_CATEGS"
self.keyTag = "TRENDINGS_TAGS"
self.keyDisc = "TRENDINGS_DISC"
self.keySigh = "TRENDINGS_SIGHT_sightings"
self.keyFalse = "TRENDINGS_SIGHT_false_positive"
#logger
logDir = cfg.get('Log', 'directory')
logfilename = cfg.get('Log', 'filename')
logPath = os.path.join(logDir, logfilename)
if not os.path.exists(logDir):
os.makedirs(logDir)
logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
self.logger = logging.getLogger(__name__)
''' SETTER '''
def addGenericTrending(self, trendingType, data, timestamp):
timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
timestampDate_str = util.getDateStrFormat(timestampDate)
keyname = "{}:{}".format(trendingType, timestampDate_str)
if isinstance(data, OrderedDict):
to_save = json.dumps(data)
else:
to_save = data
self.serv_redis_db.zincrby(keyname, to_save, 1)
self.logger.debug('Added to redis: keyname={}, content={}'.format(keyname, to_save))
def addTrendingEvent(self, eventName, timestamp):
self.addGenericTrending(self.keyEvent, eventName, timestamp)
def addTrendingCateg(self, categName, timestamp):
self.addGenericTrending(self.keyCateg, categName, timestamp)
def addTrendingDisc(self, eventName, timestamp):
self.addGenericTrending(self.keyDisc, eventName, timestamp)
def addTrendingTags(self, tags, timestamp):
for tag in tags:
ordDic = OrderedDict() #keep fields with the same layout in redis
ordDic['id'] = tag['id']
ordDic['name'] = tag['name']
ordDic['colour'] = tag['colour']
self.addGenericTrending(self.keyTag, ordDic, timestamp)
def addSightings(self, timestamp):
timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
timestampDate_str = util.getDateStrFormat(timestampDate)
keyname = "{}:{}".format(self.keySigh, timestampDate_str)
self.serv_redis_db.incrby(keyname, 1)
self.logger.debug('Incrby: keyname={}'.format(keyname))
def addFalsePositive(self, timestamp):
timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
timestampDate_str = util.getDateStrFormat(timestampDate)
keyname = "{}:{}".format(self.keyFalse, timestampDate_str)
self.serv_redis_db.incrby(keyname, 1)
self.logger.debug('Incrby: keyname={}'.format(keyname))
''' GETTER '''
def getGenericTrending(self, trendingType, dateS, dateE, topNum=0):
to_ret = []
prev_days = (dateE - dateS).days
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate))
data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True)
data = [ [record[0].decode('utf8'), record[1]] for record in data ]
data = data if data is not None else []
to_ret.append([util.getTimestamp(curDate), data])
return to_ret
def getSpecificTrending(self, trendingType, dateS, dateE, specificLabel=''):
to_ret = []
prev_days = (dateE - dateS).days
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate))
data = self.serv_redis_db.zscore(keyname, specificLabel)
data = [[specificLabel, data]] if data is not None else []
to_ret.append([util.getTimestamp(curDate), data])
return to_ret
def getTrendingEvents(self, dateS, dateE, specificLabel=None):
if specificLabel is None:
return self.getGenericTrending(self.keyEvent, dateS, dateE)
else:
specificLabel = specificLabel.replace('\\n', '\n'); # reset correctly label with their \n (CR) instead of their char value
return self.getSpecificTrending(self.keyEvent, dateS, dateE, specificLabel)
def getTrendingCategs(self, dateS, dateE):
return self.getGenericTrending(self.keyCateg, dateS, dateE)
# FIXME: Construct this when getting data
def getTrendingTags(self, dateS, dateE, topNum=12):
to_ret = []
prev_days = (dateE - dateS).days
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
keyname = "{}:{}".format(self.keyTag, util.getDateStrFormat(curDate))
data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True)
data = [ [record[0].decode('utf8'), record[1]] for record in data ]
data = data if data is not None else []
temp = []
for jText, score in data:
temp.append([json.loads(jText), score])
data = temp
to_ret.append([util.getTimestamp(curDate), data])
return to_ret
def getTrendingSightings(self, dateS, dateE):
to_ret = []
prev_days = (dateE - dateS).days
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
keyname = "{}:{}".format(self.keySigh, util.getDateStrFormat(curDate))
sight = self.serv_redis_db.get(keyname)
sight = 0 if sight is None else int(sight.decode('utf8'))
keyname = "{}:{}".format(self.keyFalse, util.getDateStrFormat(curDate))
fp = self.serv_redis_db.get(keyname)
fp = 0 if fp is None else int(fp.decode('utf8'))
to_ret.append([util.getTimestamp(curDate), { 'sightings': sight, 'false_positive': fp}])
return to_ret
def getTrendingDisc(self, dateS, dateE):
return self.getGenericTrending(self.keyDisc, dateS, dateE)
def getTypeaheadData(self, dateS, dateE):
to_ret = {}
for trendingType in [self.keyEvent, self.keyCateg]:
allSet = set()
prev_days = (dateE - dateS).days
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate))
data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True)
for elem in data:
allSet.add(elem.decode('utf8'))
to_ret[trendingType] = list(allSet)
tags = self.getTrendingTags(dateS, dateE)
tagSet = set()
for item in tags:
theDate, tagList = item
for tag in tagList:
tag = tag[0]
tagSet.add(tag['name'])
to_ret[self.keyTag] = list(tagSet)
return to_ret
# In contrary of getGenericTrending, it regroups items in the format: {item, start: timestamp1, end: timestamp2}
# so that it can be displayed easily on the timeline.
def getGenericTrendingOvertime(self, dateS, dateE, choice=None, topNum=0):
if choice == 'categs':
trendingType = self.keyCateg
elif choice == 'tags':
trendingType = self.keyTag
else:
trendingType = self.keyEvent
dico_items = {}
to_format = []
prev_days = (dateE - dateS).days
# get data
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate))
data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True)
data = [ [record[0].decode('utf8'), record[1]] for record in data ]
data = data if data is not None else []
to_format.append([util.getTimestamp(curDate), data])
for timestamp, array in to_format:
for item, _ in array:
if item not in dico_items:
dico_items[item] = []
dico_items[item].append(timestamp)
# sort timestamps in correct order
for item in dico_items.keys():
dico_items[item].sort()
# dico_items have the form: {item: [t1,t2,t4], ...}
to_ret = []
ONEDAY = 60*60*24
for item, timestamps in dico_items.items():
obj = {'name': item, 'start': timestamps[0], 'end': timestamps[0]+ONEDAY}
for t in timestamps:
if t-obj['end'] > ONEDAY: #new entry
to_ret.append(copy.deepcopy(obj))
obj['start'] = t
obj['end'] = t+ONEDAY
else: # contrinue entry
obj['end'] = t+ONEDAY
to_ret.append(obj)
return to_ret