mirror of https://github.com/MISP/misp-dashboard
GitHub
commit
f2badf7a1f
57
server.py
57
server.py
|
|
@ -11,6 +11,7 @@ import os
|
|||
|
||||
import util
|
||||
import contributor_helper
|
||||
import users_helper
|
||||
|
||||
configfile = os.path.join(os.environ['DASH_CONFIG'], 'config.cfg')
|
||||
cfg = configparser.ConfigParser()
|
||||
|
|
@ -32,6 +33,7 @@ serv_redis_db = redis.StrictRedis(
|
|||
db=cfg.getint('RedisDB', 'db'))
|
||||
|
||||
contributor_helper = contributor_helper.Contributor_helper(serv_redis_db, cfg)
|
||||
users_helper = users_helper.Users_helper(serv_redis_db, cfg)
|
||||
|
||||
subscriber_log = redis_server_log.pubsub(ignore_subscribe_messages=True)
|
||||
subscriber_log.psubscribe(cfg.get('RedisLog', 'channel'))
|
||||
|
|
@ -205,6 +207,7 @@ def users():
|
|||
return render_template('users.html',
|
||||
)
|
||||
|
||||
|
||||
''' INDEX '''
|
||||
|
||||
@app.route("/_logs")
|
||||
|
|
@ -441,5 +444,59 @@ def getTrophies():
|
|||
org = ''
|
||||
return jsonify(contributor_helper.getOrgTrophies(org))
|
||||
|
||||
|
||||
''' USERS '''
|
||||
|
||||
@app.route("/_getUserLogins")
|
||||
def getUserLogins():
|
||||
try:
|
||||
date = datetime.datetime.fromtimestamp(float(request.args.get('date')))
|
||||
except:
|
||||
date = datetime.datetime.now()
|
||||
|
||||
data = users_helper.getUserLoginsForPunchCard(date)
|
||||
return jsonify(data)
|
||||
|
||||
@app.route("/_getUserLoginsOvertime")
|
||||
def getUserLoginsOvertime():
|
||||
try:
|
||||
date = datetime.datetime.fromtimestamp(float(request.args.get('date')))
|
||||
except:
|
||||
date = datetime.datetime.now()
|
||||
|
||||
data = users_helper.getUserLoginsOvertime(date)
|
||||
return jsonify(data)
|
||||
|
||||
@app.route("/_getTopOrglogin")
|
||||
def getTopOrglogin():
|
||||
try:
|
||||
date = datetime.datetime.fromtimestamp(float(request.args.get('date')))
|
||||
except:
|
||||
date = datetime.datetime.now()
|
||||
|
||||
data = users_helper.getTopOrglogin(date, maxNum=12)
|
||||
return jsonify(data)
|
||||
|
||||
@app.route("/_getLoginVSCOntribution")
|
||||
def getLoginVSCOntribution():
|
||||
try:
|
||||
date = datetime.datetime.fromtimestamp(float(request.args.get('date')))
|
||||
except:
|
||||
date = datetime.datetime.now()
|
||||
|
||||
data = users_helper.getLoginVSCOntribution(date)
|
||||
return jsonify(data)
|
||||
|
||||
@app.route("/_getUserLoginsAndContribOvertime")
|
||||
def getUserLoginsAndContribOvertime():
|
||||
try:
|
||||
date = datetime.datetime.fromtimestamp(float(request.args.get('date')))
|
||||
except:
|
||||
date = datetime.datetime.now()
|
||||
|
||||
data = users_helper.getUserLoginsAndContribOvertime(date)
|
||||
return jsonify(data)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
app.run(host='localhost', port=8001, threaded=True)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,211 @@
|
|||
var punchcardWidget;
|
||||
var pieOrgWidget;
|
||||
var pieApiWidget;
|
||||
var overtimeWidget;
|
||||
var div_day;
|
||||
|
||||
function legendFormatter(label, series) {
|
||||
// removing unwanted "
|
||||
var label = label.replace(/\\"/g, "").replace(/\"/g, "");
|
||||
// limiting size
|
||||
if (label.length >= 50){
|
||||
labelLimited = label.substring(0, 50) + '[...]';
|
||||
} else {
|
||||
labelLimited = label;
|
||||
}
|
||||
return '<div '
|
||||
+ 'style="font-size:8pt;text-align:inherit;padding:2px;">'
|
||||
+ '<a class="tagElem" style="background-color: white; color: black;"> ' + labelLimited
|
||||
+ '</a>';
|
||||
+ '</div>';
|
||||
}
|
||||
|
||||
function highlight_punchDay() {
|
||||
if (!(div_day === undefined)) {
|
||||
div_day.removeClass('highlightDay');
|
||||
}
|
||||
|
||||
var curDay = datePickerWidgetPunch.datepicker( "getDate" ).getDay();
|
||||
if (curDay == 7)
|
||||
curDay = 0;
|
||||
div_day = $('[data-daynum="'+curDay+'"]');
|
||||
div_day.addClass('highlightDay')
|
||||
}
|
||||
|
||||
function updateDatePunch() {
|
||||
var date = datePickerWidgetPunch.datepicker( "getDate" );
|
||||
$.getJSON( url_getUserLogins+"?date="+date.getTime()/1000, function( data ) {
|
||||
if (!(punchcardWidget === undefined)) {
|
||||
punchcardWidget.settings.data = data;
|
||||
punchcardWidget.refresh();
|
||||
highlight_punchDay();
|
||||
} else {
|
||||
punchcardWidget = $('#punchcard').punchcard({
|
||||
data: data,
|
||||
singular: 'login',
|
||||
plural: 'logins',
|
||||
timezones: ['local'],
|
||||
timezoneIndex:0
|
||||
});
|
||||
punchcardWidget = punchcardWidget.data("plugin_" + "punchcard");
|
||||
highlight_punchDay();
|
||||
}
|
||||
});
|
||||
}
|
||||
function updateDatePieOrg() {
|
||||
var date = datePickerWidgetOrg.datepicker( "getDate" );
|
||||
$.getJSON( url_getTopOrglogin+"?date="+date.getTime()/1000, function( data ) {
|
||||
toPlot = [];
|
||||
for (item of data) { toPlot.push({ label: item[0], data: item[1] }); }
|
||||
if (toPlot.length == 0) {
|
||||
toPlot = [{ label: 'No data', data: 100 }];
|
||||
}
|
||||
if (!(pieOrgWidget === undefined)) {
|
||||
pieOrgWidget.setData(toPlot);
|
||||
pieOrgWidget.setupGrid();
|
||||
pieOrgWidget.draw();
|
||||
} else {
|
||||
pieOrgWidget = $.plot('#pieOrg', toPlot, {
|
||||
series: {
|
||||
pie: {
|
||||
innerRadius: 0.5,
|
||||
show: true
|
||||
}
|
||||
},
|
||||
grid: {
|
||||
hoverable: true
|
||||
}
|
||||
});
|
||||
$('#pieOrg').bind("plothover", function (event, pos, item) {
|
||||
if (item) {
|
||||
$("#tooltip").html(legendFormatter(item.series.label))
|
||||
.css({top: pos.pageY+5, left: pos.pageX+5})
|
||||
.fadeIn(200);
|
||||
} else {
|
||||
$("#tooltip").hide();
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
function updateDatePieApi() {
|
||||
var date = datePickerWidgetApi.datepicker( "getDate" );
|
||||
$.getJSON( url_getLoginVSCOntribution+"?date="+date.getTime()/1000, function( data ) {
|
||||
toPlot = [
|
||||
{label: 'Login with contribution during the day', data: data[0], color: '#4da74d' },
|
||||
{label: 'Login without contribution during the day', data:data[1], color: '#cb4b4b' }
|
||||
];
|
||||
if (data[0] == 0 && data[1] == 0) {
|
||||
toPlot = [{ label: 'No data', data: 100 }];
|
||||
}
|
||||
|
||||
if (!(pieApiWidget === undefined)) {
|
||||
pieApiWidget.setData(toPlot);
|
||||
pieApiWidget.setupGrid();
|
||||
pieApiWidget.draw();
|
||||
} else {
|
||||
pieApiWidget = $.plot('#pieApi', toPlot, {
|
||||
series: {
|
||||
pie: {
|
||||
innerRadius: 0.5,
|
||||
show: true
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
function updateDateOvertime() {
|
||||
var date = datePickerWidgetOvertime.datepicker( "getDate" );
|
||||
var now = new Date();
|
||||
if (date.toDateString() == now.toDateString()) {
|
||||
date = now;
|
||||
} else {
|
||||
date.setTime(date.getTime() + (24*60*60*1000-1)); // include data of selected date
|
||||
}
|
||||
$.getJSON( url_getUserLoginsAndContribOvertime+"?date="+parseInt(date.getTime()/1000), function( data ) {
|
||||
console.log(data);
|
||||
data_log = data['login'];
|
||||
data_contrib = data['contrib'];
|
||||
temp_log = [];
|
||||
var i=0;
|
||||
for (item of data_log) {
|
||||
var date = new Date(item[0]*1000);
|
||||
date = new Date(date.valueOf() - date.getTimezoneOffset() * 60000); // center the data around the day
|
||||
temp_log.push([date, item[1]]);
|
||||
}
|
||||
temp_contrib= [];
|
||||
var i=0;
|
||||
for (item of data_contrib) {
|
||||
var date = new Date(item[0]*1000);
|
||||
date = new Date(date.valueOf() - date.getTimezoneOffset() * 60000); // center the data around the day
|
||||
temp_contrib.push([date, item[1]]);
|
||||
}
|
||||
toPlot = [{label: 'Login', data: temp_log}, {label: 'Contribution', data: temp_contrib}];
|
||||
if (!(overtimeWidget === undefined)) {
|
||||
overtimeWidget.setData(toPlot);
|
||||
overtimeWidget.setupGrid();
|
||||
overtimeWidget.draw();
|
||||
} else {
|
||||
overtimeWidget = $.plot('#lineChart', toPlot, {
|
||||
lines: {
|
||||
show: true,
|
||||
steps: true,
|
||||
fill: true
|
||||
},
|
||||
xaxis: {
|
||||
mode: "time",
|
||||
minTickSize: [1, "day"],
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
$(document).ready(function () {
|
||||
|
||||
var datePickerOptions = {
|
||||
showOn: "button",
|
||||
maxDate: 0,
|
||||
buttonImage: urlIconCalendar,
|
||||
buttonImageOnly: true,
|
||||
buttonText: "Select date",
|
||||
showAnim: "slideDown"
|
||||
};
|
||||
// punch
|
||||
var datePickerOptionsPunch = jQuery.extend({}, datePickerOptions);
|
||||
datePickerOptionsPunch['onSelect'] = updateDatePunch;
|
||||
datePickerWidgetPunch = $( "#datepickerPunch" );
|
||||
datePickerWidgetPunch.datepicker(datePickerOptionsPunch);
|
||||
datePickerWidgetPunch.datepicker("setDate", new Date());
|
||||
// org login
|
||||
var datePickerOptionsOrg = jQuery.extend({}, datePickerOptions);
|
||||
datePickerOptionsOrg['onSelect'] = updateDatePieOrg;
|
||||
datePickerWidgetOrg = $( "#datepickerOrgLogin" );
|
||||
datePickerWidgetOrg.datepicker(datePickerOptionsOrg);
|
||||
datePickerWidgetOrg.datepicker("setDate", new Date());
|
||||
// api
|
||||
var datePickerOptionsApi = jQuery.extend({}, datePickerOptions);
|
||||
datePickerOptionsApi['onSelect'] = updateDatePieApi;
|
||||
datePickerWidgetApi = $( "#datepickerApi" );
|
||||
datePickerWidgetApi.datepicker(datePickerOptionsApi);
|
||||
datePickerWidgetApi.datepicker("setDate", new Date());
|
||||
// overtime
|
||||
var datePickerOptionsOvertime = jQuery.extend({}, datePickerOptions);
|
||||
datePickerOptionsOvertime['onSelect'] = updateDateOvertime;
|
||||
datePickerWidgetOvertime = $( "#datepickerOvertimeLogin" );
|
||||
datePickerWidgetOvertime.datepicker(datePickerOptionsOvertime);
|
||||
datePickerWidgetOvertime.datepicker("setDate", new Date());
|
||||
|
||||
updateDatePunch();
|
||||
updateDatePieOrg();
|
||||
updateDatePieApi();
|
||||
updateDateOvertime();
|
||||
|
||||
$("<div id='tooltip'></div>").css({
|
||||
position: "absolute",
|
||||
display: "none",
|
||||
}).appendTo("body");
|
||||
|
||||
});
|
||||
|
|
@ -0,0 +1,207 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
|
||||
<head>
|
||||
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<meta name="description" content="">
|
||||
<meta name="author" content="">
|
||||
|
||||
<title>MISP live dashboard</title>
|
||||
|
||||
<!-- Bootstrap Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
|
||||
<!-- Custom CSS -->
|
||||
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
|
||||
<!-- LeafLet -->
|
||||
<link rel="stylesheet" href="{{ url_for('static', filename='css/leaflet.css') }}">
|
||||
<script src="{{ url_for('static', filename='js/leaflet.js') }}"></script>
|
||||
|
||||
<!-- jQuery -->
|
||||
<script src="{{ url_for('static', filename='js/jquery.min.js') }}"></script>
|
||||
<!-- jQuery flot -->
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.pie.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.resize.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
|
||||
<!-- Bootstrap Core JavaScript -->
|
||||
<script src="{{ url_for('static', filename='js/bootstrap.js') }}"></script>
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="text/css">
|
||||
|
||||
<link rel="stylesheet" href="{{ url_for('static', filename='css/jquery-jvectormap-2.0.3.css') }}" type="text/css" media="screen"/>
|
||||
<script src="{{ url_for('static', filename='js/jquery-jvectormap-2.0.3.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery-jvectormap-world-mill.js') }}"></script>
|
||||
<link href="{{ url_for('static', filename='css/jquery-ui.min.css') }}" rel="stylesheet" type="text/css" />
|
||||
<script src="{{ url_for('static', filename='js/jquery-ui.min.js') }}"></script>
|
||||
|
||||
<link rel="stylesheet" href="{{ url_for('static', filename='css/punchcard.css') }}">
|
||||
<script src="{{ url_for('static', filename='js/moment.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/punchcard.js') }}"></script>
|
||||
|
||||
</head>
|
||||
|
||||
<style>
|
||||
|
||||
.highlightDay {
|
||||
background-color: #edc240;
|
||||
box-shadow: black 0px 0px 2px;
|
||||
}
|
||||
|
||||
.tagElem {
|
||||
display: inline-block;
|
||||
padding: 2px 4px;
|
||||
font-size: 12px;
|
||||
font-weight: bold;
|
||||
line-height: 14px;
|
||||
border-radius: 3px;
|
||||
box-shadow: 3px 3px 3px #888888;
|
||||
}
|
||||
|
||||
.panel {
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
|
||||
.col-lg-6 {
|
||||
padding-right: 5px;
|
||||
padding-left: 5px;
|
||||
}
|
||||
|
||||
.leftSepa {
|
||||
border-left-color: rgb(221, 221, 221);
|
||||
border-left-style: solid;
|
||||
border-left-width: 2px;
|
||||
padding-left: 8px;
|
||||
}
|
||||
|
||||
.textTopHeader {
|
||||
height: 50px;
|
||||
text-align: center;
|
||||
margin-left: 8px;
|
||||
float: left;
|
||||
padding-top: 9px;
|
||||
}
|
||||
|
||||
.center {
|
||||
display:block;
|
||||
margin-left:auto;
|
||||
margin-right:auto;
|
||||
}
|
||||
|
||||
small {
|
||||
font-size: 100%;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
</style>
|
||||
|
||||
<body>
|
||||
|
||||
<div id="wrapper">
|
||||
|
||||
<!-- Navigation -->
|
||||
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0; padding-left: 15px;">
|
||||
<div class="navbar-header">
|
||||
<img src="{{ url_for('static', filename='pics/MISP.png') }}" alt="MISP" style="float: left; height: 40px; margin-top: 5px;"><a class="navbar-brand dropdown-toggle" data-toggle="dropdown" href="#">MISP Users
|
||||
<span class="caret"></span></a>
|
||||
<ul class="dropdown-menu">
|
||||
<li><a href="{{ url_for('index') }}">MISP Live Dashboard</a></li>
|
||||
<li><a href="{{ url_for('geo') }}">MISP Geolocalisation</a></li>
|
||||
<li><a href="{{ url_for('contrib') }}">MISP Contributors</a></li>
|
||||
<li><a href="{{ url_for('users') }}">MISP Users</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
<!-- /.navbar-header -->
|
||||
|
||||
</nav>
|
||||
|
||||
<!-- Page Content -->
|
||||
<div id="page-wrapper" style="margin: 0px; padding: 0px;">
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
<div style="height: 10px;"></div>
|
||||
|
||||
<div class="col-lg-12">
|
||||
<div class="col-lg-7">
|
||||
<div class="panel panel-default" style="">
|
||||
<div class="panel-heading bg-info" style="font-weight: bold;">
|
||||
<b>Punch card - login</b>
|
||||
<strong class='leftSepa textTopHeader' style="float: none; padding: 11px;">Dates:
|
||||
<input type="text" id="datepickerPunch" size="10" style="">
|
||||
</strong>
|
||||
</div>
|
||||
<div id="panelbody" class="panel-body" style="">
|
||||
<div id="punchcard" style="width:100%; height: 100%;"></div>
|
||||
</div>
|
||||
</div><!-- /.panel-body -->
|
||||
</div>
|
||||
|
||||
<div class="col-lg-5">
|
||||
<div class="panel panel-default" style="">
|
||||
<div class="panel-heading bg-info" style="font-weight: bold;">
|
||||
<b>Contribution/login (last 31 days)</b>
|
||||
<strong class='leftSepa textTopHeader' style="float: none; padding: 11px;">Dates:
|
||||
<input type="text" id="datepickerOrgLogin" size="10" style="">
|
||||
</strong>
|
||||
</div>
|
||||
<div id="panelbody" class="panel-body" style="">
|
||||
<div id="pieOrg" style="width:100%; height: 20vh;"></div>
|
||||
</div>
|
||||
</div><!-- /.panel-body -->
|
||||
|
||||
<div class="panel panel-default" style="">
|
||||
<div class="panel-heading bg-info" style="font-weight: bold;">
|
||||
<b>Contribution after login</b>
|
||||
<strong class='leftSepa textTopHeader' style="float: none; padding: 11px;">Dates:
|
||||
<input type="text" id="datepickerApi" size="10" style="">
|
||||
</strong>
|
||||
</div>
|
||||
<div id="panelbody" class="panel-body" style="">
|
||||
<div id="pieApi" style="width:100%; height: 20vh;"></div>
|
||||
</div>
|
||||
</div><!-- /.panel-body -->
|
||||
</div>
|
||||
|
||||
</div><!-- /.col-lg-12 -->
|
||||
<div class="col-lg-12">
|
||||
<div class="panel panel-default" style="">
|
||||
<div class="panel-heading bg-info" style="font-weight: bold;">
|
||||
<b title="Login is unique per organisation during one hour, contribution is unique per organisation during the day">Login and contribution overtime</b>
|
||||
<strong class='leftSepa textTopHeader' style="float: none; padding: 11px;">Dates:
|
||||
<input type="text" id="datepickerOvertimeLogin" size="10" style="">
|
||||
</strong>
|
||||
</div>
|
||||
<div id="panelbody" class="panel-body" style="">
|
||||
<div id="lineChart" style="width:100%; height: 20vh;"></div>
|
||||
</div>
|
||||
</div><!-- /.panel-body -->
|
||||
</div><!-- /.col-lg-12 -->
|
||||
|
||||
</div><!-- /.row -->
|
||||
|
||||
</div> <!-- /.container-fluid -->
|
||||
|
||||
</div>
|
||||
<!-- /#page-wrapper -->
|
||||
|
||||
</div>
|
||||
<!-- /#wrapper -->
|
||||
|
||||
<!-- Index -->
|
||||
<script>
|
||||
/* URL */
|
||||
var urlIconCalendar = "{{ url_for('static', filename='pics/calendar.gif') }}";
|
||||
var url_getUserLogins = "{{ url_for('getUserLogins') }}";
|
||||
var url_getTopOrglogin = "{{ url_for('getTopOrglogin') }}";
|
||||
var url_getLoginVSCOntribution = "{{ url_for('getLoginVSCOntribution') }}";
|
||||
var url_getUserLoginsAndContribOvertime = "{{ url_for('getUserLoginsAndContribOvertime') }}";
|
||||
|
||||
/* DATA FROM CONF */
|
||||
|
||||
</script>
|
||||
|
||||
<script src="{{ url_for('static', filename='js/users.js') }}"></script>
|
||||
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
|
@ -0,0 +1,178 @@
|
|||
import math, random
|
||||
import os
|
||||
import json
|
||||
import datetime, time
|
||||
|
||||
import util
|
||||
|
||||
class Users_helper:
|
||||
def __init__(self, serv_redis_db, cfg):
|
||||
self.serv_redis_db = serv_redis_db
|
||||
self.cfg = cfg
|
||||
|
||||
def addTemporary(self, org, timestamp):
|
||||
timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
|
||||
timestampDate_str = util.getDateHoursStrFormat(timestampDate)
|
||||
keyname_timestamp = "{}:{}".format('LOGIN_TIMESTAMPSET', timestampDate_str)
|
||||
self.serv_redis_db.sadd(keyname_timestamp, org)
|
||||
self.serv_redis_db.expire(keyname_timestamp, 60*60)
|
||||
|
||||
def hasAlreadyBeenAdded(self, org, timestamp):
|
||||
timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
|
||||
timestampDate_str = util.getDateHoursStrFormat(timestampDate)
|
||||
keyname_timestamp = "{}:{}".format('LOGIN_TIMESTAMPSET', timestampDate_str)
|
||||
orgs = [ org.decode('utf8') for org in self.serv_redis_db.smembers(keyname_timestamp) ]
|
||||
if orgs is None:
|
||||
return False
|
||||
return (org in orgs)
|
||||
|
||||
def add_user_login(self, timestamp, org):
|
||||
timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
|
||||
timestampDate_str = util.getDateStrFormat(timestampDate)
|
||||
|
||||
if not self.hasAlreadyBeenAdded(org, timestamp):
|
||||
keyname_timestamp = "{}:{}".format('LOGIN_TIMESTAMP', timestampDate_str)
|
||||
self.serv_redis_db.sadd(keyname_timestamp, timestamp)
|
||||
self.addTemporary(org, timestamp)
|
||||
|
||||
keyname_org = "{}:{}".format('LOGIN_ORG', timestampDate_str)
|
||||
self.serv_redis_db.zincrby(keyname_org, org, 1)
|
||||
|
||||
def getUserLogins(self, date):
|
||||
keyname = "LOGIN_TIMESTAMP:{}".format(util.getDateStrFormat(date))
|
||||
timestamps = self.serv_redis_db.smembers(keyname)
|
||||
timestamps = [int(timestamp.decode('utf8')) for timestamp in timestamps]
|
||||
return timestamps
|
||||
|
||||
def getOrgslogin(self, date, topNum=12):
|
||||
keyname = "LOGIN_ORG:{}".format(util.getDateStrFormat(date))
|
||||
data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True)
|
||||
data = [ [record[0].decode('utf8'), record[1]] for record in data ]
|
||||
return data
|
||||
|
||||
def getAllLoggedInOrgs(self, date, prev_days=31):
|
||||
orgs = set()
|
||||
for curDate in util.getXPrevDaysSpan(date, prev_days):
|
||||
keyname = "LOGIN_ORG:{}".format(util.getDateStrFormat(curDate))
|
||||
data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=True)
|
||||
for org in data:
|
||||
orgs.add(org[0].decode('utf8'))
|
||||
return list(orgs)
|
||||
|
||||
def getOrgContribAndLogin(self, date, org, prev_days=31):
|
||||
keyname_log = "LOGIN_ORG:{}"
|
||||
keyname_contrib = "CONTRIB_DAY:{}"
|
||||
data = []
|
||||
for curDate in util.getXPrevDaysSpan(date, prev_days):
|
||||
log = self.serv_redis_db.zscore(keyname_log.format(util.getDateStrFormat(curDate)), org)
|
||||
log = 0 if log is None else 1
|
||||
contrib = self.serv_redis_db.zscore(keyname_contrib.format(util.getDateStrFormat(curDate)), org)
|
||||
contrib = 0 if contrib is None else 1
|
||||
data.append([log, contrib])
|
||||
return data
|
||||
|
||||
def getContribOverLoginScore(self, array):
|
||||
totLog = 0
|
||||
totContrib = 0
|
||||
for log, contrib in array:
|
||||
totLog += log
|
||||
totContrib += contrib
|
||||
if totLog == 0: # avoid div by 0
|
||||
totLog = 1
|
||||
return totContrib/totLog
|
||||
|
||||
def getTopOrglogin(self, date, maxNum=12, prev_days=31):
|
||||
all_logged_in_orgs = self.getAllLoggedInOrgs(date, prev_days)
|
||||
data = []
|
||||
for org in all_logged_in_orgs:
|
||||
orgStatus = self.getOrgContribAndLogin(date, org, prev_days)
|
||||
orgScore = self.getContribOverLoginScore(orgStatus)
|
||||
data.append([org, orgScore])
|
||||
data.sort(key=lambda x: x[1], reverse=True)
|
||||
return data[:maxNum]
|
||||
|
||||
|
||||
def getLoginVSCOntribution(self, date):
|
||||
keyname = "CONTRIB_DAY:{}".format(util.getDateStrFormat(date))
|
||||
orgs_contri = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=False)
|
||||
orgs_contri = [ org.decode('utf8') for org in orgs_contri ]
|
||||
orgs_login = [ org[0] for org in self.getOrgslogin(date, topNum=0) ]
|
||||
contributed_num = 0
|
||||
non_contributed_num = 0
|
||||
for org in orgs_login:
|
||||
if org in orgs_contri:
|
||||
contributed_num += 1
|
||||
else:
|
||||
non_contributed_num +=1
|
||||
return [contributed_num, non_contributed_num]
|
||||
|
||||
|
||||
def getUserLoginsForPunchCard(self, date, prev_days=6):
|
||||
week = {}
|
||||
for curDate in util.getXPrevDaysSpan(date, prev_days):
|
||||
timestamps = self.getUserLogins(curDate)
|
||||
day = {}
|
||||
for timestamp in timestamps:
|
||||
date = datetime.datetime.fromtimestamp(float(timestamp))
|
||||
if date.hour not in day:
|
||||
day[date.hour] = 0
|
||||
day[date.hour] += 1
|
||||
week[curDate.weekday()] = day
|
||||
|
||||
# Format data
|
||||
data = []
|
||||
for d in range(7):
|
||||
try:
|
||||
to_append = []
|
||||
for h in range(24):
|
||||
try:
|
||||
to_append.append(week[d][h])
|
||||
except KeyError:
|
||||
to_append.append(0)
|
||||
data.append(to_append)
|
||||
except KeyError: # no data
|
||||
data.append([0 for x in range(24)])
|
||||
# swap: punchcard day starts on monday
|
||||
data = [data[6]]+data[:6]
|
||||
return data
|
||||
|
||||
def getUserLoginsAndContribOvertime(self, date, prev_days=6):
|
||||
dico_hours_contrib = {}
|
||||
dico_hours = {}
|
||||
for curDate in util.getXPrevHoursSpan(date, prev_days*24):
|
||||
dico_hours[util.getTimestamp(curDate)] = 0 # populate with empty data
|
||||
dico_hours_contrib[util.getTimestamp(curDate)] = 0 # populate with empty data
|
||||
|
||||
for curDate in util.getXPrevDaysSpan(date, prev_days):
|
||||
timestamps = self.getUserLogins(curDate)
|
||||
keyname = "CONTRIB_DAY:{}".format(util.getDateStrFormat(curDate))
|
||||
|
||||
orgs_contri = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=False)
|
||||
orgs_contri_num = len(orgs_contri)
|
||||
for curDate in util.getHoursSpanOfDate(curDate, adaptToFitCurrentTime=True): #fill hole day
|
||||
dico_hours_contrib[util.getTimestamp(curDate)] = orgs_contri_num
|
||||
|
||||
for timestamp in timestamps: # sum occurence during the current hour
|
||||
dateTimestamp = datetime.datetime.fromtimestamp(float(timestamp))
|
||||
dateTimestamp = dateTimestamp.replace(minute=0, second=0, microsecond=0)
|
||||
try:
|
||||
dico_hours[util.getTimestamp(dateTimestamp)] += 1
|
||||
except KeyError: # timestamp out of bound (greater than 1 week)
|
||||
pass
|
||||
|
||||
# Format data
|
||||
# login
|
||||
to_ret = {}
|
||||
data = []
|
||||
for curDate, occ in dico_hours.items():
|
||||
data.append([curDate, occ])
|
||||
data.sort(key=lambda x: x[0])
|
||||
to_ret['login'] = data
|
||||
# contrib
|
||||
data = []
|
||||
for curDate, occ in dico_hours_contrib.items():
|
||||
data.append([curDate, occ])
|
||||
data.sort(key=lambda x: x[0])
|
||||
to_ret['contrib'] = data
|
||||
|
||||
return to_ret
|
||||
28
util.py
28
util.py
|
|
@ -24,8 +24,34 @@ def getXPrevDaysSpan(date, days):
|
|||
to_return.append(de - datetime.timedelta(days=i))
|
||||
return to_return
|
||||
|
||||
def getXPrevHoursSpan(date, hours):
|
||||
de = date
|
||||
de = de.replace(minute=0, second=0, microsecond=0)
|
||||
ds = de - datetime.timedelta(hours=hours)
|
||||
|
||||
delta = de - ds
|
||||
to_return = []
|
||||
for i in range(0, int(delta.total_seconds()/3600)+1):
|
||||
to_return.append(de - datetime.timedelta(hours=i))
|
||||
return to_return
|
||||
|
||||
def getHoursSpanOfDate(date, adaptToFitCurrentTime=True, daySpanned=6):
|
||||
ds = date
|
||||
ds = ds.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
to_return = []
|
||||
now = datetime.datetime.now()
|
||||
for i in range(0, 24):
|
||||
the_date = ds + datetime.timedelta(hours=i)
|
||||
if the_date > now or the_date < now - datetime.timedelta(days=daySpanned): # avoid going outside
|
||||
continue
|
||||
to_return.append(the_date)
|
||||
return to_return
|
||||
|
||||
def getDateStrFormat(date):
|
||||
return str(date.year)+str(date.month).zfill(2)+str(date.day).zfill(2)
|
||||
|
||||
def getDateHoursStrFormat(date):
|
||||
return getDateStrFormat(date)+str(date.hour)
|
||||
|
||||
def getTimestamp(date):
|
||||
return time.mktime(date.timetuple())
|
||||
return int(time.mktime(date.timetuple()))
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ import geoip2.database
|
|||
|
||||
import util
|
||||
import contributor_helper
|
||||
import users_helper
|
||||
|
||||
configfile = os.path.join(os.environ['DASH_CONFIG'], 'config.cfg')
|
||||
cfg = configparser.ConfigParser()
|
||||
|
|
@ -53,6 +54,7 @@ serv_redis_db = redis.StrictRedis(
|
|||
db=cfg.getint('RedisDB', 'db'))
|
||||
|
||||
contributor_helper = contributor_helper.Contributor_helper(serv_redis_db, cfg)
|
||||
users_helper = users_helper.Users_helper(serv_redis_db, cfg)
|
||||
|
||||
reader = geoip2.database.Reader(PATH_TO_DB)
|
||||
|
||||
|
|
@ -140,6 +142,9 @@ def handleContribution(zmq_name, org, contribType, categ, action, pntMultiplier=
|
|||
nowSec = int(time.time())
|
||||
pnts_to_add = DEFAULT_PNTS_REWARD
|
||||
|
||||
# if there is a contribution, there is a login (even if ti comes from the API)
|
||||
users_helper.add_user_login(nowSec, org)
|
||||
|
||||
# is a valid contribution
|
||||
if categ is not None:
|
||||
try:
|
||||
|
|
@ -187,6 +192,17 @@ def handler_keepalive(zmq_name, jsonevent):
|
|||
to_push = [ jsonevent['uptime'] ]
|
||||
publish_log(zmq_name, 'Keepalive', to_push)
|
||||
|
||||
def handler_user(zmq_name, jsondata):
|
||||
json_user = jsondata['User']
|
||||
userID = json_user['id']
|
||||
org = userID
|
||||
try: #only consider user login
|
||||
timestamp = json_user['current_login']
|
||||
except KeyError:
|
||||
return
|
||||
if timestamp != 0: # "invited_by": "xxxx" ???
|
||||
users_helper.add_user_login(timestamp, org)
|
||||
|
||||
def handler_conversation(zmq_name, jsonevent):
|
||||
try: #only consider POST, not THREAD
|
||||
jsonpost = jsonevent['Post']
|
||||
|
|
@ -323,7 +339,7 @@ dico_action = {
|
|||
"misp_json_object": handler_object,
|
||||
"misp_json_sighting": handler_sighting,
|
||||
"misp_json_organisation": handler_log,
|
||||
"misp_json_user": handler_log,
|
||||
"misp_json_user": handler_user,
|
||||
"misp_json_conversation": handler_conversation
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue