mirror of https://github.com/MISP/misp-dashboard
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
124 lines
3.8 KiB
124 lines
3.8 KiB
import datetime |
|
import time |
|
from collections import defaultdict |
|
|
|
ONE_DAY = 60*60*24 |
|
|
|
def getZrange(serv_redis_db, keyCateg, date, topNum, endSubkey=""): |
|
date_str = getDateStrFormat(date) |
|
keyname = "{}:{}{}".format(keyCateg, date_str, endSubkey) |
|
data = serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True) |
|
data = [ [record[0], record[1]] for record in data ] |
|
return data |
|
|
|
def noSpaceLower(text): |
|
return text.lower().replace(' ', '_') |
|
|
|
def push_to_redis_zset(serv_redis_db, mainKey, toAdd, endSubkey="", count=1): |
|
now = datetime.datetime.now() |
|
today_str = getDateStrFormat(now) |
|
keyname = "{}:{}{}".format(mainKey, today_str, endSubkey) |
|
serv_redis_db.zincrby(keyname, count, toAdd) |
|
|
|
def getMonthSpan(date): |
|
ds = datetime.datetime(date.year, date.month, 1) |
|
dyear = 1 if ds.month+1 > 12 else 0 |
|
dmonth = -12 if ds.month+1 > 12 else 0 |
|
de = datetime.datetime(ds.year + dyear, ds.month+1 + dmonth, 1) |
|
|
|
delta = de - ds |
|
to_return = [] |
|
for i in range(delta.days): |
|
to_return.append(ds + datetime.timedelta(days=i)) |
|
return to_return |
|
|
|
def getXPrevDaysSpan(date, days): |
|
de = date |
|
ds = de - datetime.timedelta(days=days) |
|
|
|
delta = de - ds |
|
to_return = [] |
|
for i in range(delta.days+1): |
|
to_return.append(de - datetime.timedelta(days=i)) |
|
return to_return |
|
|
|
def getXPrevHoursSpan(date, hours): |
|
de = date |
|
de = de.replace(minute=0, second=0, microsecond=0) |
|
ds = de - datetime.timedelta(hours=hours) |
|
|
|
delta = de - ds |
|
to_return = [] |
|
for i in range(0, int(delta.total_seconds()/3600)+1): |
|
to_return.append(de - datetime.timedelta(hours=i)) |
|
return to_return |
|
|
|
def getHoursSpanOfDate(date, adaptToFitCurrentTime=True, daySpanned=6): |
|
ds = date |
|
ds = ds.replace(hour=0, minute=0, second=0, microsecond=0) |
|
to_return = [] |
|
now = datetime.datetime.now() |
|
for i in range(0, 24): |
|
the_date = ds + datetime.timedelta(hours=i) |
|
if the_date > now or the_date < now - datetime.timedelta(days=daySpanned): # avoid going outside |
|
continue |
|
to_return.append(the_date) |
|
return to_return |
|
|
|
def getDateStrFormat(date): |
|
return str(date.year)+str(date.month).zfill(2)+str(date.day).zfill(2) |
|
|
|
def getDateHoursStrFormat(date): |
|
return getDateStrFormat(date)+str(date.hour) |
|
|
|
def getTimestamp(date): |
|
return int(time.mktime(date.timetuple())) |
|
|
|
|
|
def sortByTrendingScore(toSort, topNum=5): |
|
scoredLabels = defaultdict(float) |
|
numDay = len(toSort) |
|
baseDecay = 1.0 |
|
decayRate = lambda x: baseDecay*((numDay-x**2)/numDay) |
|
|
|
for i, arr in enumerate(toSort): |
|
timestamp = arr[0] |
|
dailyData = arr[1] |
|
for item in dailyData: |
|
label = item[0] |
|
occ = item[1] |
|
scoredLabels[label] += occ*decayRate(i) |
|
|
|
topList = [[l, s] for l, s in scoredLabels.items()] |
|
topList.sort(key=lambda x: x[1], reverse=True) |
|
topSet = [ l for l, v in topList[:topNum]] |
|
|
|
# now that we have the top, filter out poor scored elements |
|
topArray = [] |
|
for arr in toSort: |
|
timestamp = arr[0] |
|
dailyData = arr[1] |
|
topDailyArray = list(filter(lambda item: (item[0] in topSet), dailyData)) |
|
dailyCombi = [timestamp, topDailyArray] |
|
topArray.append(dailyCombi) |
|
|
|
return topArray |
|
|
|
|
|
def getFields(obj, fields): |
|
jsonWalker = fields.split('.') |
|
itemToExplore = obj |
|
lastName = "" |
|
try: |
|
for i in jsonWalker: |
|
itemToExplore = itemToExplore[i] |
|
lastName = i |
|
if type(itemToExplore) is list: |
|
return {'name': lastName, 'data': itemToExplore} |
|
else: |
|
if i == 'timestamp': |
|
itemToExplore = datetime.datetime.utcfromtimestamp( |
|
int(itemToExplore)).strftime('%Y-%m-%d %H:%M:%S') |
|
return itemToExplore |
|
except KeyError as e: |
|
return None
|
|
|