mirror of https://github.com/MISP/misp-dashboard
125 lines
3.8 KiB
Python
125 lines
3.8 KiB
Python
import datetime
|
|
import time
|
|
from collections import defaultdict
|
|
|
|
ONE_DAY = 60*60*24
|
|
|
|
def getZrange(serv_redis_db, keyCateg, date, topNum, endSubkey=""):
|
|
date_str = getDateStrFormat(date)
|
|
keyname = "{}:{}{}".format(keyCateg, date_str, endSubkey)
|
|
data = serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True)
|
|
data = [ [record[0], record[1]] for record in data ]
|
|
return data
|
|
|
|
def noSpaceLower(text):
|
|
return text.lower().replace(' ', '_')
|
|
|
|
def push_to_redis_zset(serv_redis_db, mainKey, toAdd, endSubkey="", count=1):
|
|
now = datetime.datetime.now()
|
|
today_str = getDateStrFormat(now)
|
|
keyname = "{}:{}{}".format(mainKey, today_str, endSubkey)
|
|
serv_redis_db.zincrby(keyname, count, toAdd)
|
|
|
|
def getMonthSpan(date):
|
|
ds = datetime.datetime(date.year, date.month, 1)
|
|
dyear = 1 if ds.month+1 > 12 else 0
|
|
dmonth = -12 if ds.month+1 > 12 else 0
|
|
de = datetime.datetime(ds.year + dyear, ds.month+1 + dmonth, 1)
|
|
|
|
delta = de - ds
|
|
to_return = []
|
|
for i in range(delta.days):
|
|
to_return.append(ds + datetime.timedelta(days=i))
|
|
return to_return
|
|
|
|
def getXPrevDaysSpan(date, days):
|
|
de = date
|
|
ds = de - datetime.timedelta(days=days)
|
|
|
|
delta = de - ds
|
|
to_return = []
|
|
for i in range(delta.days+1):
|
|
to_return.append(de - datetime.timedelta(days=i))
|
|
return to_return
|
|
|
|
def getXPrevHoursSpan(date, hours):
|
|
de = date
|
|
de = de.replace(minute=0, second=0, microsecond=0)
|
|
ds = de - datetime.timedelta(hours=hours)
|
|
|
|
delta = de - ds
|
|
to_return = []
|
|
for i in range(0, int(delta.total_seconds()/3600)+1):
|
|
to_return.append(de - datetime.timedelta(hours=i))
|
|
return to_return
|
|
|
|
def getHoursSpanOfDate(date, adaptToFitCurrentTime=True, daySpanned=6):
|
|
ds = date
|
|
ds = ds.replace(hour=0, minute=0, second=0, microsecond=0)
|
|
to_return = []
|
|
now = datetime.datetime.now()
|
|
for i in range(0, 24):
|
|
the_date = ds + datetime.timedelta(hours=i)
|
|
if the_date > now or the_date < now - datetime.timedelta(days=daySpanned): # avoid going outside
|
|
continue
|
|
to_return.append(the_date)
|
|
return to_return
|
|
|
|
def getDateStrFormat(date):
|
|
return str(date.year)+str(date.month).zfill(2)+str(date.day).zfill(2)
|
|
|
|
def getDateHoursStrFormat(date):
|
|
return getDateStrFormat(date)+str(date.hour)
|
|
|
|
def getTimestamp(date):
|
|
return int(time.mktime(date.timetuple()))
|
|
|
|
|
|
def sortByTrendingScore(toSort, topNum=5):
|
|
scoredLabels = defaultdict(float)
|
|
numDay = len(toSort)
|
|
baseDecay = 1.0
|
|
decayRate = lambda x: baseDecay*((numDay-x**2)/numDay)
|
|
|
|
for i, arr in enumerate(toSort):
|
|
timestamp = arr[0]
|
|
dailyData = arr[1]
|
|
for item in dailyData:
|
|
label = item[0]
|
|
occ = item[1]
|
|
scoredLabels[label] += occ*decayRate(i)
|
|
|
|
topList = [[l, s] for l, s in scoredLabels.items()]
|
|
topList.sort(key=lambda x: x[1], reverse=True)
|
|
topSet = [ l for l, v in topList[:topNum]]
|
|
|
|
# now that we have the top, filter out poor scored elements
|
|
topArray = []
|
|
for arr in toSort:
|
|
timestamp = arr[0]
|
|
dailyData = arr[1]
|
|
topDailyArray = list(filter(lambda item: (item[0] in topSet), dailyData))
|
|
dailyCombi = [timestamp, topDailyArray]
|
|
topArray.append(dailyCombi)
|
|
|
|
return topArray
|
|
|
|
|
|
def getFields(obj, fields):
|
|
jsonWalker = fields.split('.')
|
|
itemToExplore = obj
|
|
lastName = ""
|
|
try:
|
|
for i in jsonWalker:
|
|
itemToExplore = itemToExplore[i]
|
|
lastName = i
|
|
if type(itemToExplore) is list:
|
|
return {'name': lastName, 'data': itemToExplore}
|
|
else:
|
|
if i == 'timestamp':
|
|
itemToExplore = datetime.datetime.utcfromtimestamp(
|
|
int(itemToExplore)).strftime('%Y-%m-%d %H:%M:%S')
|
|
return itemToExplore
|
|
except KeyError as e:
|
|
return None
|