2017-12-11 14:21:33 +01:00
|
|
|
import copy
|
2019-05-29 01:42:39 +02:00
|
|
|
import datetime
|
|
|
|
import json
|
2017-12-04 16:44:44 +01:00
|
|
|
import logging
|
2019-05-29 01:42:39 +02:00
|
|
|
import math
|
|
|
|
import os
|
|
|
|
import random
|
2019-06-01 08:08:48 +02:00
|
|
|
import sys
|
2019-05-29 01:42:39 +02:00
|
|
|
import time
|
2017-11-17 14:45:20 +01:00
|
|
|
from collections import OrderedDict
|
2017-11-17 10:07:24 +01:00
|
|
|
|
|
|
|
import util
|
|
|
|
|
2019-05-29 01:42:39 +02:00
|
|
|
|
2017-11-17 10:07:24 +01:00
|
|
|
class Trendings_helper:
|
|
|
|
def __init__(self, serv_redis_db, cfg):
|
|
|
|
self.serv_redis_db = serv_redis_db
|
|
|
|
self.cfg = cfg
|
|
|
|
|
2017-11-29 16:46:45 +01:00
|
|
|
# REDIS keys
|
|
|
|
self.keyEvent = "TRENDINGS_EVENTS"
|
|
|
|
self.keyCateg = "TRENDINGS_CATEGS"
|
|
|
|
self.keyTag = "TRENDINGS_TAGS"
|
|
|
|
self.keyDisc = "TRENDINGS_DISC"
|
|
|
|
self.keySigh = "TRENDINGS_SIGHT_sightings"
|
|
|
|
self.keyFalse = "TRENDINGS_SIGHT_false_positive"
|
|
|
|
|
2017-12-05 10:23:40 +01:00
|
|
|
#logger
|
|
|
|
logDir = cfg.get('Log', 'directory')
|
2019-05-29 03:41:43 +02:00
|
|
|
logfilename = cfg.get('Log', 'helpers_filename')
|
2017-12-05 10:23:40 +01:00
|
|
|
logPath = os.path.join(logDir, logfilename)
|
|
|
|
if not os.path.exists(logDir):
|
|
|
|
os.makedirs(logDir)
|
2019-05-29 03:41:43 +02:00
|
|
|
try:
|
|
|
|
logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
|
|
|
|
except PermissionError as error:
|
|
|
|
print(error)
|
|
|
|
print("Please fix the above and try again.")
|
|
|
|
sys.exit(126)
|
2017-12-05 10:23:40 +01:00
|
|
|
self.logger = logging.getLogger(__name__)
|
|
|
|
|
2017-11-17 10:07:24 +01:00
|
|
|
''' SETTER '''
|
|
|
|
|
|
|
|
def addGenericTrending(self, trendingType, data, timestamp):
|
|
|
|
timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
|
|
|
|
timestampDate_str = util.getDateStrFormat(timestampDate)
|
|
|
|
keyname = "{}:{}".format(trendingType, timestampDate_str)
|
2017-11-17 15:32:47 +01:00
|
|
|
if isinstance(data, OrderedDict):
|
|
|
|
to_save = json.dumps(data)
|
|
|
|
else:
|
|
|
|
to_save = data
|
2019-06-18 11:25:17 +02:00
|
|
|
self.serv_redis_db.zincrby(keyname, 1, to_save)
|
2017-12-05 10:23:40 +01:00
|
|
|
self.logger.debug('Added to redis: keyname={}, content={}'.format(keyname, to_save))
|
2017-11-17 10:07:24 +01:00
|
|
|
|
|
|
|
def addTrendingEvent(self, eventName, timestamp):
|
2017-11-29 16:46:45 +01:00
|
|
|
self.addGenericTrending(self.keyEvent, eventName, timestamp)
|
2017-11-17 10:07:24 +01:00
|
|
|
|
|
|
|
def addTrendingCateg(self, categName, timestamp):
|
2017-11-29 16:46:45 +01:00
|
|
|
self.addGenericTrending(self.keyCateg, categName, timestamp)
|
2017-11-17 10:07:24 +01:00
|
|
|
|
2017-11-17 16:50:04 +01:00
|
|
|
def addTrendingDisc(self, eventName, timestamp):
|
2017-11-29 16:46:45 +01:00
|
|
|
self.addGenericTrending(self.keyDisc, eventName, timestamp)
|
2017-11-17 16:50:04 +01:00
|
|
|
|
2017-11-17 10:07:24 +01:00
|
|
|
def addTrendingTags(self, tags, timestamp):
|
|
|
|
for tag in tags:
|
2017-11-17 14:45:20 +01:00
|
|
|
ordDic = OrderedDict() #keep fields with the same layout in redis
|
|
|
|
ordDic['id'] = tag['id']
|
|
|
|
ordDic['name'] = tag['name']
|
|
|
|
ordDic['colour'] = tag['colour']
|
2017-11-29 16:46:45 +01:00
|
|
|
self.addGenericTrending(self.keyTag, ordDic, timestamp)
|
2017-11-17 10:07:24 +01:00
|
|
|
|
2017-11-17 11:46:51 +01:00
|
|
|
def addSightings(self, timestamp):
|
|
|
|
timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
|
|
|
|
timestampDate_str = util.getDateStrFormat(timestampDate)
|
2017-11-29 16:46:45 +01:00
|
|
|
keyname = "{}:{}".format(self.keySigh, timestampDate_str)
|
2017-11-17 11:46:51 +01:00
|
|
|
self.serv_redis_db.incrby(keyname, 1)
|
2017-12-05 10:23:40 +01:00
|
|
|
self.logger.debug('Incrby: keyname={}'.format(keyname))
|
2017-11-17 11:46:51 +01:00
|
|
|
|
|
|
|
def addFalsePositive(self, timestamp):
|
|
|
|
timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
|
|
|
|
timestampDate_str = util.getDateStrFormat(timestampDate)
|
2017-11-29 16:46:45 +01:00
|
|
|
keyname = "{}:{}".format(self.keyFalse, timestampDate_str)
|
2017-11-17 11:46:51 +01:00
|
|
|
self.serv_redis_db.incrby(keyname, 1)
|
2017-12-05 10:23:40 +01:00
|
|
|
self.logger.debug('Incrby: keyname={}'.format(keyname))
|
2017-11-17 10:07:24 +01:00
|
|
|
|
|
|
|
''' GETTER '''
|
|
|
|
|
2018-09-24 09:21:02 +02:00
|
|
|
def getGenericTrending(self, trendingType, dateS, dateE, topNum=10):
|
2017-11-17 10:07:24 +01:00
|
|
|
to_ret = []
|
|
|
|
prev_days = (dateE - dateS).days
|
|
|
|
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
|
|
|
|
keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate))
|
2018-09-24 09:21:02 +02:00
|
|
|
data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=True)
|
2019-06-19 11:32:06 +02:00
|
|
|
data = [ [record[0], record[1]] for record in data ]
|
2017-11-17 10:07:24 +01:00
|
|
|
data = data if data is not None else []
|
|
|
|
to_ret.append([util.getTimestamp(curDate), data])
|
2018-09-24 09:21:02 +02:00
|
|
|
to_ret = util.sortByTrendingScore(to_ret, topNum=topNum)
|
2017-11-17 10:07:24 +01:00
|
|
|
return to_ret
|
|
|
|
|
2017-11-21 08:42:54 +01:00
|
|
|
def getSpecificTrending(self, trendingType, dateS, dateE, specificLabel=''):
|
|
|
|
to_ret = []
|
|
|
|
prev_days = (dateE - dateS).days
|
|
|
|
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
|
|
|
|
keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate))
|
|
|
|
data = self.serv_redis_db.zscore(keyname, specificLabel)
|
|
|
|
data = [[specificLabel, data]] if data is not None else []
|
|
|
|
to_ret.append([util.getTimestamp(curDate), data])
|
|
|
|
return to_ret
|
|
|
|
|
2018-09-24 09:21:02 +02:00
|
|
|
def getTrendingEvents(self, dateS, dateE, specificLabel=None, topNum=None):
|
2017-11-21 08:42:54 +01:00
|
|
|
if specificLabel is None:
|
2018-09-24 09:21:02 +02:00
|
|
|
return self.getGenericTrending(self.keyEvent, dateS, dateE, topNum=topNum)
|
2017-11-21 08:42:54 +01:00
|
|
|
else:
|
2017-11-21 14:49:41 +01:00
|
|
|
specificLabel = specificLabel.replace('\\n', '\n'); # reset correctly label with their \n (CR) instead of their char value
|
2017-11-29 16:46:45 +01:00
|
|
|
return self.getSpecificTrending(self.keyEvent, dateS, dateE, specificLabel)
|
2017-11-17 10:07:24 +01:00
|
|
|
|
2018-09-24 10:26:15 +02:00
|
|
|
def getTrendingCategs(self, dateS, dateE, topNum=None):
|
|
|
|
return self.getGenericTrending(self.keyCateg, dateS, dateE, topNum=topNum)
|
2017-11-17 10:07:24 +01:00
|
|
|
|
2017-11-20 17:42:25 +01:00
|
|
|
# FIXME: Construct this when getting data
|
2017-11-17 14:45:20 +01:00
|
|
|
def getTrendingTags(self, dateS, dateE, topNum=12):
|
|
|
|
to_ret = []
|
|
|
|
prev_days = (dateE - dateS).days
|
|
|
|
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
|
2017-11-29 16:46:45 +01:00
|
|
|
keyname = "{}:{}".format(self.keyTag, util.getDateStrFormat(curDate))
|
2017-11-17 14:45:20 +01:00
|
|
|
data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True)
|
2019-06-19 11:32:06 +02:00
|
|
|
data = [ [record[0], record[1]] for record in data ]
|
2017-11-17 14:45:20 +01:00
|
|
|
data = data if data is not None else []
|
|
|
|
temp = []
|
|
|
|
for jText, score in data:
|
|
|
|
temp.append([json.loads(jText), score])
|
|
|
|
data = temp
|
|
|
|
to_ret.append([util.getTimestamp(curDate), data])
|
|
|
|
return to_ret
|
|
|
|
|
|
|
|
def getTrendingSightings(self, dateS, dateE):
|
|
|
|
to_ret = []
|
|
|
|
prev_days = (dateE - dateS).days
|
|
|
|
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
|
2017-11-29 16:46:45 +01:00
|
|
|
keyname = "{}:{}".format(self.keySigh, util.getDateStrFormat(curDate))
|
2017-11-17 14:45:20 +01:00
|
|
|
sight = self.serv_redis_db.get(keyname)
|
2019-06-19 11:32:06 +02:00
|
|
|
sight = 0 if sight is None else int(sight)
|
2017-11-29 16:46:45 +01:00
|
|
|
keyname = "{}:{}".format(self.keyFalse, util.getDateStrFormat(curDate))
|
2017-11-17 14:45:20 +01:00
|
|
|
fp = self.serv_redis_db.get(keyname)
|
2019-06-19 11:32:06 +02:00
|
|
|
fp = 0 if fp is None else int(fp)
|
2017-11-17 14:45:20 +01:00
|
|
|
to_ret.append([util.getTimestamp(curDate), { 'sightings': sight, 'false_positive': fp}])
|
|
|
|
return to_ret
|
2017-11-17 16:50:04 +01:00
|
|
|
|
2018-09-24 10:26:15 +02:00
|
|
|
def getTrendingDisc(self, dateS, dateE, topNum=None):
|
|
|
|
return self.getGenericTrending(self.keyDisc, dateS, dateE, topNum=topNum)
|
2017-11-20 17:42:25 +01:00
|
|
|
|
|
|
|
def getTypeaheadData(self, dateS, dateE):
|
|
|
|
to_ret = {}
|
2017-11-29 16:46:45 +01:00
|
|
|
for trendingType in [self.keyEvent, self.keyCateg]:
|
2017-11-20 17:42:25 +01:00
|
|
|
allSet = set()
|
|
|
|
prev_days = (dateE - dateS).days
|
|
|
|
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
|
|
|
|
keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate))
|
|
|
|
data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True)
|
|
|
|
for elem in data:
|
2019-06-19 11:32:06 +02:00
|
|
|
allSet.add(elem)
|
2017-11-20 17:42:25 +01:00
|
|
|
to_ret[trendingType] = list(allSet)
|
|
|
|
tags = self.getTrendingTags(dateS, dateE)
|
|
|
|
tagSet = set()
|
|
|
|
for item in tags:
|
|
|
|
theDate, tagList = item
|
|
|
|
for tag in tagList:
|
|
|
|
tag = tag[0]
|
|
|
|
tagSet.add(tag['name'])
|
2017-11-29 16:46:45 +01:00
|
|
|
to_ret[self.keyTag] = list(tagSet)
|
2017-11-20 17:42:25 +01:00
|
|
|
return to_ret
|
2017-12-11 12:19:11 +01:00
|
|
|
|
|
|
|
# In contrary of getGenericTrending, it regroups items in the format: {item, start: timestamp1, end: timestamp2}
|
|
|
|
# so that it can be displayed easily on the timeline.
|
2017-12-11 14:54:22 +01:00
|
|
|
def getGenericTrendingOvertime(self, dateS, dateE, choice=None, topNum=0):
|
2017-12-18 15:31:22 +01:00
|
|
|
if choice == 'categs':
|
|
|
|
trendingType = self.keyCateg
|
|
|
|
elif choice == 'tags':
|
2017-12-11 14:54:22 +01:00
|
|
|
trendingType = self.keyTag
|
|
|
|
else:
|
|
|
|
trendingType = self.keyEvent
|
|
|
|
|
2017-12-11 12:19:11 +01:00
|
|
|
dico_items = {}
|
|
|
|
to_format = []
|
|
|
|
prev_days = (dateE - dateS).days
|
|
|
|
# get data
|
|
|
|
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
|
|
|
|
keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate))
|
|
|
|
data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True)
|
2019-06-19 11:32:06 +02:00
|
|
|
data = [ [record[0], record[1]] for record in data ]
|
2017-12-11 12:19:11 +01:00
|
|
|
data = data if data is not None else []
|
|
|
|
to_format.append([util.getTimestamp(curDate), data])
|
|
|
|
|
|
|
|
for timestamp, array in to_format:
|
|
|
|
for item, _ in array:
|
|
|
|
if item not in dico_items:
|
|
|
|
dico_items[item] = []
|
|
|
|
dico_items[item].append(timestamp)
|
|
|
|
|
|
|
|
# sort timestamps in correct order
|
|
|
|
for item in dico_items.keys():
|
|
|
|
dico_items[item].sort()
|
|
|
|
# dico_items have the form: {item: [t1,t2,t4], ...}
|
|
|
|
to_ret = []
|
|
|
|
ONEDAY = 60*60*24
|
|
|
|
for item, timestamps in dico_items.items():
|
2017-12-11 15:42:25 +01:00
|
|
|
obj = {'name': item, 'start': timestamps[0], 'end': timestamps[0]+ONEDAY}
|
2017-12-11 12:19:11 +01:00
|
|
|
for t in timestamps:
|
|
|
|
if t-obj['end'] > ONEDAY: #new entry
|
2017-12-11 14:21:33 +01:00
|
|
|
to_ret.append(copy.deepcopy(obj))
|
2017-12-11 15:42:25 +01:00
|
|
|
obj['start'] = t
|
|
|
|
obj['end'] = t+ONEDAY
|
2017-12-11 12:19:11 +01:00
|
|
|
else: # contrinue entry
|
2017-12-11 15:42:25 +01:00
|
|
|
obj['end'] = t+ONEDAY
|
2017-12-11 12:19:11 +01:00
|
|
|
to_ret.append(obj)
|
|
|
|
return to_ret
|