mirror of https://github.com/MISP/misp-dashboard
GitHub
commit
8578e81f3d
59
README.md
59
README.md
|
|
@ -1,19 +1,38 @@
|
|||
# MISP-Dashboard
|
||||
An experimental dashboard showing live data and statistics from the ZMQ of one or more MISP instances.
|
||||
|
||||
|
||||
# Installation
|
||||
- Launch ```./install_dependencies.sh``` from the MISP-Dashboard directory
|
||||
- Update the configuration file ```config.cfg``` so that it matches your system
|
||||
- Fields that you may change:
|
||||
- RedisGlobal -> host
|
||||
- RedisGlobal -> port
|
||||
- RedisGlobal -> zmq_url
|
||||
- RedisGlobal -> misp_web_url
|
||||
- RedisMap -> pathMaxMindDB
|
||||
|
||||
# Starting the System
|
||||
- Activate your virtualenv ```. ./DASHENV/bin/activate```
|
||||
- Listen to the MISP feed by starting the zmq_subscriber ```./zmq_subscriber.py```
|
||||
- Start the Flask server ```./server.py```
|
||||
- Access the interface at ```http://localhost:8001/```
|
||||
|
||||
# Features
|
||||
|
||||
## Live Dashboard
|
||||
- Possibility to subscribe to multiple ZMQ feeds
|
||||
- Shows direct contribution made by organisations
|
||||
- Shows live resolvable posted locations
|
||||
|
||||
|
||||
|
||||
|
||||
## Geolocalisation Dashboard
|
||||
|
||||
- Provides historical geolocalised information to support security teams, CSIRTs or SOC finding threats in their constituency
|
||||
- Possibility to get geospatial information from specific regions
|
||||
|
||||
|
||||
|
||||
|
||||
## Contributors Dashboard
|
||||
|
||||
|
|
@ -29,25 +48,25 @@ __Includes__:
|
|||
- Gamification of the platform:
|
||||
- Two different levels of ranking with unique icons
|
||||
- Exclusive obtainable badges for source code contributors and donator
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# Installation
|
||||
- Launch ```./install_dependencies.sh``` from the MISP-Dashboard directory
|
||||
- Update the configuration file ```config.cfg``` so that it matches your system
|
||||
- Fields that you may change:
|
||||
- RedisGlobal -> host
|
||||
- RedisGlobal -> port
|
||||
- RedisGlobal -> zmq_url
|
||||
- RedisGlobal -> misp_web_url
|
||||
|
||||
# Starting the System
|
||||
- Activate your virtualenv ```. ./DASHENV/bin/activate```
|
||||
- Listen to the MISP feed by starting the zmq_subscriber ```./zmq_subscriber.py```
|
||||
- Start the Flask server ```./server.py```
|
||||
- Access the interface at ```http://localhost:8001/```
|
||||
## Users Dashboard
|
||||
|
||||
- Shows when and how the platform is used:
|
||||
- Login punchcard and overtime
|
||||
- Contribution vs login
|
||||
|
||||
|
||||
|
||||
## Trendings Dashboard
|
||||
|
||||
- Provides real time information to support security teams, CSIRTs or SOC showing current threats and activity
|
||||
- Shows most active events, categories and tags
|
||||
- Shows sightings and discussion overtime
|
||||
|
||||
|
||||
|
||||
# zmq_subscriber options
|
||||
```usage: zmq_subscriber.py [-h] [-n ZMQNAME] [-u ZMQURL]
|
||||
|
|
@ -62,7 +81,7 @@ optional arguments:
|
|||
The URL to connect to
|
||||
```
|
||||
|
||||
## License
|
||||
# License
|
||||
Images and logos are handmade for:
|
||||
- rankingMISPOrg/
|
||||
- rankingMISPMonthly/
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ rankMultiplier = 2
|
|||
categories_in_datatable = ["internal_reference", "targeting_data", "antivirus_detection", "payload_delivery", "artifacts_dropped", "payload_installation", "persistence_mechanism", "network_activity", "payload_type", "attribution", "external_analysis", "financial_fraud", "support_Tool", "social_network", "person", "other" ]
|
||||
default_pnts_per_contribution = 1
|
||||
# array of the form [[category, pntsRcv], ...]
|
||||
pnts_per_contribution = [["payload_delivery", 1], ["artifact_dropped", 1], ["network_activity", 1]]
|
||||
pnts_per_contribution = [["payload_delivery", 1], ["artifacts_dropped", 1], ["network_activity", 1]]
|
||||
additional_help_text = ["Sightings multiplies earned points by 2", "Editing an attribute earns you the same as creating one"]
|
||||
|
||||
[Log]
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
set -e
|
||||
set -x
|
||||
|
||||
sudo apt-get install python3-virtualenv -y
|
||||
sudo apt-get install python3-virtualenv virtualenv screen redis-server -y
|
||||
|
||||
if [ -z "$VIRTUAL_ENV" ]; then
|
||||
virtualenv -p python3 DASHENV
|
||||
|
|
@ -102,4 +102,10 @@ wget https://cdn.datatables.net/${DATATABLE_VERSION}/js/dataTables.bootstrap.js
|
|||
git clone https://github.com/bassjobsen/Bootstrap-3-Typeahead.git temp/Bootstrap-3-Typeahead
|
||||
mv temp/Bootstrap-3-Typeahead/bootstrap3-typeahead.min.js ./static/js
|
||||
|
||||
#punchcard
|
||||
git clone https://github.com/melenaos/jquery-punchcard.git temp/jquery-punchcard
|
||||
mv temp/jquery-punchcard/src/punchcard.js ./static/js
|
||||
mv temp/jquery-punchcard/src/punchcard.css ./static/css
|
||||
wget https://momentjs.com/downloads/moment.js -O ./static/js/moment.js
|
||||
|
||||
rm -rf ./temp
|
||||
|
|
|
|||
Binary file not shown.
|
After Width: | Height: | Size: 358 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 100 KiB |
90
server.py
90
server.py
|
|
@ -12,6 +12,7 @@ import os
|
|||
import util
|
||||
import contributor_helper
|
||||
import users_helper
|
||||
import trendings_helper
|
||||
|
||||
configfile = os.path.join(os.environ['DASH_CONFIG'], 'config.cfg')
|
||||
cfg = configparser.ConfigParser()
|
||||
|
|
@ -34,6 +35,7 @@ serv_redis_db = redis.StrictRedis(
|
|||
|
||||
contributor_helper = contributor_helper.Contributor_helper(serv_redis_db, cfg)
|
||||
users_helper = users_helper.Users_helper(serv_redis_db, cfg)
|
||||
trendings_helper = trendings_helper.Trendings_helper(serv_redis_db, cfg)
|
||||
|
||||
subscriber_log = redis_server_log.pubsub(ignore_subscribe_messages=True)
|
||||
subscriber_log.psubscribe(cfg.get('RedisLog', 'channel'))
|
||||
|
|
@ -208,6 +210,18 @@ def users():
|
|||
)
|
||||
|
||||
|
||||
@app.route("/trendings")
|
||||
def trendings():
|
||||
maxNum = request.args.get('maxNum')
|
||||
try:
|
||||
maxNum = int(maxNum)
|
||||
except:
|
||||
maxNum = 15
|
||||
|
||||
return render_template('trendings.html',
|
||||
maxNum=maxNum
|
||||
)
|
||||
|
||||
''' INDEX '''
|
||||
|
||||
@app.route("/_logs")
|
||||
|
|
@ -497,6 +511,82 @@ def getUserLoginsAndContribOvertime():
|
|||
data = users_helper.getUserLoginsAndContribOvertime(date)
|
||||
return jsonify(data)
|
||||
|
||||
''' TRENDINGS '''
|
||||
@app.route("/_getTrendingEvents")
|
||||
def getTrendingEvents():
|
||||
try:
|
||||
dateS = datetime.datetime.fromtimestamp(float(request.args.get('dateS')))
|
||||
dateE = datetime.datetime.fromtimestamp(float(request.args.get('dateE')))
|
||||
except:
|
||||
dateS = datetime.datetime.now() - datetime.timedelta(days=7)
|
||||
dateE = datetime.datetime.now()
|
||||
|
||||
specificLabel = request.args.get('specificLabel')
|
||||
data = trendings_helper.getTrendingEvents(dateS, dateE, specificLabel)
|
||||
return jsonify(data)
|
||||
|
||||
@app.route("/_getTrendingCategs")
|
||||
def getTrendingCategs():
|
||||
try:
|
||||
dateS = datetime.datetime.fromtimestamp(float(request.args.get('dateS')))
|
||||
dateE = datetime.datetime.fromtimestamp(float(request.args.get('dateE')))
|
||||
except:
|
||||
dateS = datetime.datetime.now() - datetime.timedelta(days=7)
|
||||
dateE = datetime.datetime.now()
|
||||
|
||||
|
||||
data = trendings_helper.getTrendingCategs(dateS, dateE)
|
||||
return jsonify(data)
|
||||
|
||||
@app.route("/_getTrendingTags")
|
||||
def getTrendingTags():
|
||||
try:
|
||||
dateS = datetime.datetime.fromtimestamp(float(request.args.get('dateS')))
|
||||
dateE = datetime.datetime.fromtimestamp(float(request.args.get('dateE')))
|
||||
except:
|
||||
dateS = datetime.datetime.now() - datetime.timedelta(days=7)
|
||||
dateE = datetime.datetime.now()
|
||||
|
||||
|
||||
data = trendings_helper.getTrendingTags(dateS, dateE)
|
||||
return jsonify(data)
|
||||
|
||||
@app.route("/_getTrendingSightings")
|
||||
def getTrendingSightings():
|
||||
try:
|
||||
dateS = datetime.datetime.fromtimestamp(float(request.args.get('dateS')))
|
||||
dateE = datetime.datetime.fromtimestamp(float(request.args.get('dateE')))
|
||||
except:
|
||||
dateS = datetime.datetime.now() - datetime.timedelta(days=7)
|
||||
dateE = datetime.datetime.now()
|
||||
|
||||
data = trendings_helper.getTrendingSightings(dateS, dateE)
|
||||
return jsonify(data)
|
||||
|
||||
@app.route("/_getTrendingDisc")
|
||||
def getTrendingDisc():
|
||||
try:
|
||||
dateS = datetime.datetime.fromtimestamp(float(request.args.get('dateS')))
|
||||
dateE = datetime.datetime.fromtimestamp(float(request.args.get('dateE')))
|
||||
except:
|
||||
dateS = datetime.datetime.now() - datetime.timedelta(days=7)
|
||||
dateE = datetime.datetime.now()
|
||||
|
||||
|
||||
data = trendings_helper.getTrendingDisc(dateS, dateE)
|
||||
return jsonify(data)
|
||||
|
||||
@app.route("/_getTypeaheadData")
|
||||
def getTypeaheadData():
|
||||
try:
|
||||
dateS = datetime.datetime.fromtimestamp(float(request.args.get('dateS')))
|
||||
dateE = datetime.datetime.fromtimestamp(float(request.args.get('dateE')))
|
||||
except:
|
||||
dateS = datetime.datetime.now() - datetime.timedelta(days=7)
|
||||
dateE = datetime.datetime.now()
|
||||
|
||||
data = trendings_helper.getTypeaheadData(dateS, dateE)
|
||||
return jsonify(data)
|
||||
|
||||
if __name__ == '__main__':
|
||||
app.run(host='localhost', port=8001, threaded=True)
|
||||
|
|
|
|||
|
|
@ -90,6 +90,11 @@
|
|||
display: none;
|
||||
}
|
||||
|
||||
.table > thead > tr > th.centerCell {
|
||||
text-align: center;
|
||||
min-width: 45px;
|
||||
}
|
||||
|
||||
.table > tbody > tr > td.centerCell {
|
||||
text-align: center;
|
||||
min-width: 45px;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,441 @@
|
|||
/* VARS */
|
||||
var dateStart;
|
||||
var dateEnd;
|
||||
var eventPie = ["#eventPie"];
|
||||
var eventLine = ["#eventLine"];
|
||||
var categPie = ["#categPie"];
|
||||
var categLine = ["#categLine"];
|
||||
var tagPie = ["#tagPie"];
|
||||
var tagLine = ["#tagLine"];
|
||||
var sightingLineWidget;
|
||||
var discLine = ["#discussionLine"];
|
||||
var allData;
|
||||
var globalColorMapping = {};
|
||||
|
||||
/* OPTIONS */
|
||||
var datePickerOptions = {
|
||||
showOn: "button",
|
||||
maxDate: 0,
|
||||
buttonImage: urlIconCalendar,
|
||||
buttonImageOnly: true,
|
||||
buttonText: "Select date",
|
||||
showAnim: "slideDown",
|
||||
onSelect: dateChanged
|
||||
};
|
||||
var lineChartOption = {
|
||||
lines: {
|
||||
show: true,
|
||||
},
|
||||
points: { show: true },
|
||||
xaxis: {
|
||||
mode: "time",
|
||||
minTickSize: [1, "day"],
|
||||
},
|
||||
legend: { show: false },
|
||||
grid: {
|
||||
hoverable: true
|
||||
}
|
||||
};
|
||||
var pieChartOption = {
|
||||
series: {
|
||||
pie: {
|
||||
innerRadius: 0.2,
|
||||
show: true,
|
||||
radius: 100,
|
||||
label: {
|
||||
show: true,
|
||||
radius: 6/10,
|
||||
formatter: innerPieLabelFormatter,
|
||||
}
|
||||
}
|
||||
},
|
||||
legend: {
|
||||
show: true,
|
||||
labelFormatter: legendFormatter
|
||||
},
|
||||
grid: {
|
||||
hoverable: true,
|
||||
clickable: true
|
||||
}
|
||||
};
|
||||
var typeaheadOption_event = {
|
||||
source: function (query, process) {
|
||||
if (allData === undefined) { // caching
|
||||
return $.getJSON(url_getTypeaheadData, function (data) {
|
||||
allData = data;
|
||||
return process(data.TRENDINGS_EVENTS);
|
||||
});
|
||||
} else {
|
||||
return process(allData.TRENDINGS_EVENTS);
|
||||
}
|
||||
},
|
||||
updater: function(theevent) {
|
||||
updateLineForLabel(eventLine, theevent, undefined, url_getTrendingEvent);
|
||||
}
|
||||
}
|
||||
var typeaheadOption_categ = {
|
||||
source: function (query, process) {
|
||||
if (allData === undefined) { // caching
|
||||
return $.getJSON(url_getTypeaheadData, function (data) {
|
||||
allData = data;
|
||||
return process(data.TRENDINGS_CATEGS);
|
||||
});
|
||||
} else {
|
||||
return process(allData.TRENDINGS_CATEGS);
|
||||
}
|
||||
},
|
||||
updater: function(categ) {
|
||||
updateLineForLabel(categLine, categ, undefined, url_getTrendingCateg);
|
||||
}
|
||||
}
|
||||
var typeaheadOption_tag = {
|
||||
source: function (query, process) {
|
||||
if (allData === undefined) { // caching
|
||||
return $.getJSON(url_getTypeaheadData, function (data) {
|
||||
allData = data;
|
||||
return process(data.TRENDINGS_TAGS);
|
||||
});
|
||||
} else {
|
||||
return process(allData.TRENDINGS_TAGS);
|
||||
}
|
||||
},
|
||||
updater: function(tag) {
|
||||
updateLineForLabel(tagLine, tag, undefined, url_getTrendingTag);
|
||||
}
|
||||
}
|
||||
|
||||
/* FUNCTIONS */
|
||||
function getColor(label) {
|
||||
try {
|
||||
return globalColorMapping[label];
|
||||
} catch(err) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function innerPieLabelFormatter(label, series) {
|
||||
var count = series.data[0][1];
|
||||
return '<div '
|
||||
+ 'style="font-size:8pt;text-align:inherit;padding:2px;">'
|
||||
+ '<a class="tagElem" style="background-color: '+ 'white' + ';'
|
||||
+ 'color: ' + 'black' + ';"> ' + count + '</a>'
|
||||
+ '</div>';
|
||||
}
|
||||
|
||||
function getTextColour(rgb) {
|
||||
var r = parseInt('0x'+rgb.substring(0,2));
|
||||
var g = parseInt('0x'+rgb.substring(2,4));
|
||||
var b = parseInt('0x'+rgb.substring(4,6));
|
||||
var avg = ((2 * r) + b + (3 * g))/6;
|
||||
if (avg < 128) {
|
||||
return 'white';
|
||||
} else {
|
||||
return 'black';
|
||||
}
|
||||
}
|
||||
function legendFormatter(label, series) {
|
||||
try {
|
||||
jsonLabel = JSON.parse(label);
|
||||
var backgroundColor = jsonLabel.colour;
|
||||
var color = getTextColour(backgroundColor.substring(1,6));;
|
||||
var labelText = jsonLabel.name;
|
||||
return '<div '
|
||||
+ 'style="font-size:8pt;text-align:inherit;padding:2px;">'
|
||||
+ '<a class="tagElem" style="background-color: '+ backgroundColor + ';'
|
||||
+ 'color: ' + color + ';"> ' + labelText + '</a>'
|
||||
+ '</div>';
|
||||
} catch(err) {
|
||||
// removing unwanted "
|
||||
var label = label.replace(/\\"/g, "").replace(/\"/g, "");
|
||||
// limiting size
|
||||
if (label.length >= 40){
|
||||
labelLimited = label.substring(0, 40) + '[...]';
|
||||
} else {
|
||||
labelLimited = label;
|
||||
}
|
||||
return '<div '
|
||||
+ 'style="font-size:8pt;text-align:inherit;padding:2px;">'
|
||||
+ '<a class="tagElem" style="background-color: white; color: black;"> ' + labelLimited
|
||||
+ '</a>';
|
||||
+ '</div>';
|
||||
}
|
||||
}
|
||||
|
||||
function generateEmptyAndFillData(data, specificLabel, colorMapping) {
|
||||
// formating - Generate empty data
|
||||
var toPlot_obj = {};
|
||||
var allDates = [];
|
||||
for (var arr of data) {
|
||||
var date = new Date(arr[0]*1000);
|
||||
date = new Date(date.valueOf() - date.getTimezoneOffset() * 60000); // center the data around the day
|
||||
allDates.push(date);
|
||||
var items = arr[1];
|
||||
if (items.length > 0) {
|
||||
for(var item_arr of items) {
|
||||
var count = item_arr[1];
|
||||
var itemStr = JSON.stringify(item_arr[0]);
|
||||
if (specificLabel === undefined || specificLabel == item_arr[0]) { // no tag
|
||||
if(toPlot_obj[itemStr] === undefined)
|
||||
toPlot_obj[itemStr] = {};
|
||||
toPlot_obj[itemStr][date] = count;
|
||||
} else if (specificLabel == item_arr[0].name) { // tag
|
||||
if(toPlot_obj[itemStr] === undefined)
|
||||
toPlot_obj[itemStr] = {};
|
||||
toPlot_obj[itemStr][date] = count;
|
||||
} else if (specificLabel == itemStr.substring(1, itemStr.length-1)) { // tag from click (countain { and }, need to supress it)
|
||||
if(toPlot_obj[itemStr] === undefined)
|
||||
toPlot_obj[itemStr] = {};
|
||||
toPlot_obj[itemStr][date] = count;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
toPlot = []
|
||||
for (var itemStr in toPlot_obj) {
|
||||
if (toPlot_obj.hasOwnProperty(itemStr)) {
|
||||
data_toPlot = []
|
||||
for (var curDate of allDates) {
|
||||
if (toPlot_obj[itemStr].hasOwnProperty(curDate)) {
|
||||
data_toPlot.push([curDate, toPlot_obj[itemStr][curDate]])
|
||||
} else {
|
||||
data_toPlot.push([curDate, 0])
|
||||
}
|
||||
}
|
||||
if (colorMapping === undefined) {
|
||||
//try to get color, else no color
|
||||
var colorCode = getColor(itemStr);
|
||||
if (!( colorCode === undefined)) {
|
||||
toPlot.push({label: itemStr, data: data_toPlot, color: colorCode})
|
||||
} else {
|
||||
toPlot.push({label: itemStr, data: data_toPlot})
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
var color = colorMapping[itemStr].colour;
|
||||
toPlot.push({label: itemStr, data: data_toPlot, color: color})
|
||||
} catch(err) {
|
||||
// ignore, only shows data displayed in the pie chart
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return toPlot;
|
||||
}
|
||||
|
||||
function compareObj(a,b) {
|
||||
if (a.data < b.data)
|
||||
return -1;
|
||||
if (a.data > b.data)
|
||||
return 1;
|
||||
return 0;
|
||||
}
|
||||
/* UPDATES */
|
||||
|
||||
// return the color maping: label->color
|
||||
function updatePie(pie, line, data, url) {
|
||||
var pieID = pie[0];
|
||||
var pieWidget = pie[1];
|
||||
var itemMapping = {};
|
||||
var colorMapping = {};
|
||||
if (data === undefined || data.length == 0 || (data[0] == 0 && data[1] == 0)) {
|
||||
toPlot = [{ label: 'No data', data: 100 }];
|
||||
} else {
|
||||
toPlot_obj = {}
|
||||
for (var arr of data) {
|
||||
var date = arr[0];
|
||||
var items = arr[1]
|
||||
for(var item_arr of items) {
|
||||
var itemStr = JSON.stringify(item_arr[0]);
|
||||
itemMapping[itemStr] = item_arr[0];
|
||||
var count = item_arr[1];
|
||||
if(toPlot_obj[itemStr] === undefined)
|
||||
toPlot_obj[itemStr] = 0;
|
||||
toPlot_obj[itemStr] += count;
|
||||
}
|
||||
}
|
||||
if (Object.keys(toPlot_obj).length == 0) { // no data
|
||||
toPlot = [{ label: 'No data', data: 100 }];
|
||||
} else {
|
||||
toPlot = [];
|
||||
for (var itemStr in toPlot_obj) {
|
||||
if (toPlot_obj.hasOwnProperty(itemStr)) {
|
||||
var itemColor = itemMapping[itemStr].colour
|
||||
colorMapping[itemStr] = itemColor;
|
||||
toPlot.push({label: itemStr, data: toPlot_obj[itemStr], color: itemColor})
|
||||
}
|
||||
}
|
||||
}
|
||||
toPlot.sort(compareObj).reverse();
|
||||
var maxNum = $('#num_selector').val();
|
||||
toPlot = toPlot.slice(0,maxNum); // take at max 12 elements
|
||||
}
|
||||
if (!(pieWidget === undefined)) {
|
||||
pieWidget.setData(toPlot);
|
||||
pieWidget.setupGrid();
|
||||
pieWidget.draw();
|
||||
// fill colorMapping
|
||||
for (item of pieWidget.getData()) {
|
||||
colorMapping[item.label] = {colour: item.color};
|
||||
}
|
||||
} else {
|
||||
pieWidget = $.plot(pieID, toPlot, pieChartOption);
|
||||
pie.push(pieWidget);
|
||||
// Hover
|
||||
$(pieID).bind("plothover", function (event, pos, item) {
|
||||
if (item) {
|
||||
$("#tooltip").html(legendFormatter(item.series.label))
|
||||
.css({top: pos.pageY+5, left: pos.pageX+5})
|
||||
.fadeIn(200);
|
||||
} else {
|
||||
$("#tooltip").hide();
|
||||
}
|
||||
});
|
||||
// Click
|
||||
$(pieID).bind("plotclick", function(event, pos, obj) {
|
||||
if (!obj) { return; }
|
||||
var specificLabel = obj.series.label;
|
||||
colorMapping[specificLabel] = {};
|
||||
colorMapping[specificLabel] = { colour: obj.series.color };
|
||||
updateLineForLabel(line, specificLabel.substring(1, specificLabel.length-1), colorMapping, url);
|
||||
});
|
||||
for (item of pieWidget.getData()) {
|
||||
colorMapping[item.label] = {colour: item.color};
|
||||
}
|
||||
}
|
||||
return colorMapping;
|
||||
}
|
||||
|
||||
function updateLine(line, data, chartOptions, specificLabel, colorMapping) {
|
||||
lineID = line[0];
|
||||
lineWidget = line[1];
|
||||
toPlot = generateEmptyAndFillData(data, specificLabel, colorMapping);
|
||||
// plot
|
||||
if (!(lineWidget === undefined)) {
|
||||
lineWidget.setData(toPlot);
|
||||
lineWidget.setupGrid();
|
||||
lineWidget.draw();
|
||||
} else {
|
||||
if (chartOptions === undefined) {
|
||||
chartOptions = lineChartOption;
|
||||
}
|
||||
lineWidget = $.plot(lineID, toPlot, chartOptions);
|
||||
line.push(lineWidget);
|
||||
$(lineID).bind("plothover", function (event, pos, item) {
|
||||
if (item) {
|
||||
$("#tooltip").html(legendFormatter(item.series.label))
|
||||
.css({top: item.pageY+5, left: item.pageX+5})
|
||||
.fadeIn(200);
|
||||
} else {
|
||||
$("#tooltip").hide();
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function updateSignthingsChart() {
|
||||
$.getJSON( url_getTrendingSightings+"?dateS="+parseInt(dateStart.getTime()/1000)+"&dateE="+parseInt(dateEnd.getTime()/1000), function( data ) {
|
||||
var toPlot_obj = {};
|
||||
toPlot_obj['Sightings'] = [];
|
||||
toPlot_obj['False positive'] = [];
|
||||
var allDates = [];
|
||||
for (var arr of data) {
|
||||
var date = new Date(arr[0]*1000);
|
||||
date = new Date(date.valueOf() - date.getTimezoneOffset() * 60000); // center the data around the day
|
||||
allDates.push(date);
|
||||
var items = arr[1];
|
||||
var sight = items.sightings;
|
||||
var fp = items.false_positive;
|
||||
toPlot_obj['Sightings'].push([date, sight]);
|
||||
toPlot_obj['False positive'].push([date, -fp]);
|
||||
}
|
||||
toPlot = []
|
||||
toPlot.push({label: 'Sightings', data: toPlot_obj['Sightings'], color: '#4da74d'})
|
||||
toPlot.push({label: 'False positive', data: toPlot_obj['False positive'], color: '#cb4b4b'})
|
||||
|
||||
if (!(sightingLineWidget === undefined)) {
|
||||
sightingLineWidget.setData(toPlot);
|
||||
sightingLineWidget.setupGrid();
|
||||
sightingLineWidget.draw();
|
||||
} else {
|
||||
var lineChartOptionSight = jQuery.extend(true, {}, lineChartOption);
|
||||
lineChartOptionSight['legend']['show'] = true;
|
||||
lineChartOptionSight['legend']['position'] = 'nw';
|
||||
lineChartOptionSight['grid'] = {};
|
||||
lineChartOptionSight['lines']['fill'] = true;
|
||||
sightingLineWidget = $.plot("#sightingLine", toPlot, lineChartOptionSight);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function updateLineForLabel(line, specificLabel, colorMapping, url) {
|
||||
$.getJSON( url+"?dateS="+parseInt(dateStart.getTime()/1000)+"&dateE="+parseInt(dateEnd.getTime()/1000)+"&specificLabel="+specificLabel, function( data ) {
|
||||
updateLine(line, data, undefined, specificLabel, colorMapping);
|
||||
});
|
||||
}
|
||||
|
||||
function updatePieLine(pie, line, url) {
|
||||
$.getJSON( url+"?dateS="+parseInt(dateStart.getTime()/1000)+"&dateE="+parseInt(dateEnd.getTime()/1000), function( data ) {
|
||||
var colorMapping = updatePie(pie, line, data, url);
|
||||
for (var item in colorMapping) {
|
||||
if (colorMapping.hasOwnProperty(item) && colorMapping[item] != undefined) {
|
||||
globalColorMapping[item] = colorMapping[item].colour;
|
||||
}
|
||||
}
|
||||
updateLine(line, data, undefined, undefined, colorMapping);
|
||||
});
|
||||
}
|
||||
|
||||
function updateDisc() {
|
||||
var lineChartOptionDisc = jQuery.extend(true, {}, lineChartOption);
|
||||
lineChartOptionDisc['legend']['show'] = true;
|
||||
lineChartOptionDisc['legend']['position'] = 'nw';
|
||||
lineChartOptionDisc['lines']['fill'] = true;
|
||||
$.getJSON( url_getTrendingDisc+"?dateS="+parseInt(dateStart.getTime()/1000)+"&dateE="+parseInt(dateEnd.getTime()/1000), function( data ) {
|
||||
updateLine(discLine, data, lineChartOptionDisc);
|
||||
});
|
||||
}
|
||||
|
||||
function dateChanged() {
|
||||
dateStart = datePickerWidgetStart.datepicker( "getDate" );
|
||||
dateEnd = datePickerWidgetEnd.datepicker( "getDate" );
|
||||
updatePieLine(eventPie, eventLine, url_getTrendingEvent);
|
||||
updatePieLine(categPie, categLine, url_getTrendingCateg);
|
||||
updatePieLine(tagPie, tagLine, url_getTrendingTag);
|
||||
updateSignthingsChart();
|
||||
updateDisc();
|
||||
}
|
||||
|
||||
$(document).ready(function () {
|
||||
|
||||
datePickerWidgetStart = $( "#datepickerStart" ).datepicker(datePickerOptions);
|
||||
var lastWeekDate = new Date();
|
||||
lastWeekDate.setDate(lastWeekDate.getDate()-7);
|
||||
datePickerWidgetStart.datepicker("setDate", lastWeekDate);
|
||||
dateStart = datePickerWidgetStart.datepicker( "getDate" );
|
||||
datePickerWidgetEnd = $( "#datepickerEnd" ).datepicker(datePickerOptions);
|
||||
datePickerWidgetEnd.datepicker("setDate", new Date());
|
||||
dateEnd = datePickerWidgetEnd.datepicker( "getDate" );
|
||||
|
||||
$('#typeaheadEvent').typeahead(typeaheadOption_event);
|
||||
$('#typeaheadCateg').typeahead(typeaheadOption_categ);
|
||||
$('#typeaheadTag').typeahead(typeaheadOption_tag);
|
||||
|
||||
updatePieLine(eventPie, eventLine, url_getTrendingEvent)
|
||||
updatePieLine(categPie, categLine, url_getTrendingCateg)
|
||||
updatePieLine(tagPie, tagLine, url_getTrendingTag)
|
||||
updateSignthingsChart();
|
||||
updateDisc();
|
||||
|
||||
$( "#num_selector" ).change(function() {
|
||||
var sel = parseInt($( this ).val());
|
||||
var maxNum = sel;
|
||||
window.location.href = url_currentPage+'?maxNum='+maxNum;
|
||||
});
|
||||
|
||||
$("<div id='tooltip'></div>").css({
|
||||
position: "absolute",
|
||||
display: "none",
|
||||
}).appendTo("body");
|
||||
|
||||
});
|
||||
|
|
@ -8,7 +8,7 @@
|
|||
<meta name="description" content="">
|
||||
<meta name="author" content="">
|
||||
|
||||
<title>MISP live dashboard</title>
|
||||
<title>MISP Contributors</title>
|
||||
|
||||
<!-- Bootstrap Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
|
||||
|
|
@ -188,14 +188,14 @@
|
|||
<thead>
|
||||
<tr>
|
||||
{% for title in trophy_title_str %}
|
||||
<th>{{ title }}</th>
|
||||
<th class="centerCell">{{ title }}</th>
|
||||
{% endfor %}
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id='bodyTableTrophyModal'>
|
||||
<tr>
|
||||
{% for perc in trophy_mapping %}
|
||||
<td>{{ perc }}</td>
|
||||
<td class="centerCell">{{ perc }}</td>
|
||||
{% endfor %}
|
||||
</tr>
|
||||
<tr>
|
||||
|
|
@ -249,6 +249,8 @@
|
|||
<li><a href="{{ url_for('index') }}">MISP Live Dashboard</a></li>
|
||||
<li><a href="{{ url_for('geo') }}">MISP Geolocalisation</a></li>
|
||||
<li><a href="{{ url_for('contrib') }}">MISP Contributors</a></li>
|
||||
<li><a href="{{ url_for('users') }}">MISP Users</a></li>
|
||||
<li><a href="{{ url_for('trendings') }}">MISP Trendings</a></li>
|
||||
</ul>
|
||||
<div id="ledsHolder" style="float: left; height: 50px;">
|
||||
<div class='leftSepa textTopHeader'>
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
<meta name="description" content="">
|
||||
<meta name="author" content="">
|
||||
|
||||
<title>MISP live dashboard</title>
|
||||
<title>MISP Geolocalisation</title>
|
||||
|
||||
<!-- Bootstrap Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
|
||||
|
|
@ -91,6 +91,8 @@ small {
|
|||
<li><a href="{{ url_for('index') }}">MISP Live Dashboard</a></li>
|
||||
<li><a href="{{ url_for('geo') }}">MISP Geolocalisation</a></li>
|
||||
<li><a href="{{ url_for('contrib') }}">MISP Contributors</a></li>
|
||||
<li><a href="{{ url_for('users') }}">MISP Users</a></li>
|
||||
<li><a href="{{ url_for('trendings') }}">MISP Trendings</a></li>
|
||||
</ul>
|
||||
|
||||
<div id="ledsHolder" style="float: right; height: 50px;">
|
||||
|
|
|
|||
|
|
@ -133,6 +133,8 @@ small {
|
|||
<li><a href="{{ url_for('index') }}">MISP Live Dashboard</a></li>
|
||||
<li><a href="{{ url_for('geo') }}">MISP Geolocalisation</a></li>
|
||||
<li><a href="{{ url_for('contrib') }}">MISP Contributors</a></li>
|
||||
<li><a href="{{ url_for('users') }}">MISP Users</a></li>
|
||||
<li><a href="{{ url_for('trendings') }}">MISP Trendings</a></li>
|
||||
</ul>
|
||||
|
||||
<div id="ledsHolder" style="float: right; height: 50px;"></div>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,240 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
|
||||
<head>
|
||||
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<meta name="description" content="">
|
||||
<meta name="author" content="">
|
||||
|
||||
<title>MISP Trendings</title>
|
||||
|
||||
<!-- Bootstrap Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
|
||||
<!-- Custom CSS -->
|
||||
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
|
||||
<!-- LeafLet -->
|
||||
<link rel="stylesheet" href="{{ url_for('static', filename='css/leaflet.css') }}">
|
||||
<script src="{{ url_for('static', filename='js/leaflet.js') }}"></script>
|
||||
|
||||
<!-- jQuery -->
|
||||
<script src="{{ url_for('static', filename='js/jquery.min.js') }}"></script>
|
||||
<!-- jQuery flot -->
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.pie.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.resize.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
|
||||
<!-- Bootstrap Core JavaScript -->
|
||||
<script src="{{ url_for('static', filename='js/bootstrap.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap3-typeahead.min.js') }}"></script>
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="text/css">
|
||||
|
||||
<link rel="stylesheet" href="{{ url_for('static', filename='css/jquery-jvectormap-2.0.3.css') }}" type="text/css" media="screen"/>
|
||||
<script src="{{ url_for('static', filename='js/jquery-jvectormap-2.0.3.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery-jvectormap-world-mill.js') }}"></script>
|
||||
<link href="{{ url_for('static', filename='css/jquery-ui.min.css') }}" rel="stylesheet" type="text/css" />
|
||||
<script src="{{ url_for('static', filename='js/jquery-ui.min.js') }}"></script>
|
||||
|
||||
</head>
|
||||
|
||||
<style>
|
||||
|
||||
.highlightDay {
|
||||
background-color: #edc240;
|
||||
box-shadow: black 0px 0px 2px;
|
||||
}
|
||||
|
||||
.tagElem {
|
||||
display: inline-block;
|
||||
padding: 2px 4px;
|
||||
font-size: 12px;
|
||||
font-weight: bold;
|
||||
line-height: 14px;
|
||||
border-radius: 3px;
|
||||
box-shadow: 3px 3px 3px #888888;
|
||||
}
|
||||
|
||||
.panel {
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
|
||||
.panel-body {
|
||||
padding: 0px;
|
||||
}
|
||||
|
||||
.col-lg-6 {
|
||||
padding-right: 5px;
|
||||
padding-left: 5px;
|
||||
}
|
||||
|
||||
.leftSepa {
|
||||
border-left-color: rgb(221, 221, 221);
|
||||
border-left-style: solid;
|
||||
border-left-width: 2px;
|
||||
padding-left: 8px;
|
||||
}
|
||||
|
||||
.textTopHeader {
|
||||
height: 50px;
|
||||
text-align: center;
|
||||
margin-left: 8px;
|
||||
float: left;
|
||||
padding-top: 9px;
|
||||
}
|
||||
|
||||
.center {
|
||||
display:block;
|
||||
margin-left:auto;
|
||||
margin-right:auto;
|
||||
}
|
||||
|
||||
small {
|
||||
font-size: 100%;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
</style>
|
||||
|
||||
<body>
|
||||
|
||||
<div id="wrapper">
|
||||
|
||||
<!-- Navigation -->
|
||||
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0; padding-left: 15px;">
|
||||
<div class="navbar-header">
|
||||
<img src="{{ url_for('static', filename='pics/MISP.png') }}" alt="MISP" style="float: left; height: 40px; margin-top: 5px;"><a class="navbar-brand dropdown-toggle" data-toggle="dropdown" href="#">MISP Trendings
|
||||
<span class="caret"></span></a>
|
||||
<ul class="dropdown-menu">
|
||||
<li><a href="{{ url_for('index') }}">MISP Live Dashboard</a></li>
|
||||
<li><a href="{{ url_for('geo') }}">MISP Geolocalisation</a></li>
|
||||
<li><a href="{{ url_for('contrib') }}">MISP Contributors</a></li>
|
||||
<li><a href="{{ url_for('users') }}">MISP Users</a></li>
|
||||
<li><a href="{{ url_for('trendings') }}">MISP Trendings</a></li>
|
||||
</ul>
|
||||
<div class='leftSepa textTopHeader'>
|
||||
<strong>Max display:
|
||||
<select id="num_selector">
|
||||
<option value="1" {% if maxNum == 1 %}selected=""{% endif %}>1</option>
|
||||
<option value="3" {% if maxNum == 3 %}selected=""{% endif %}>3</option>
|
||||
<option value="5" {% if maxNum == 5 %}selected=""{% endif %}>5</option>
|
||||
<option value="10" {% if maxNum == 10 %}selected=""{% endif %}>10</option>
|
||||
<option value="15" {% if maxNum == 15 %}selected=""{% endif %}>15</option>
|
||||
<option value="20" {% if maxNum == 20 %}selected=""{% endif %}>20</option>
|
||||
</select>
|
||||
</strong>
|
||||
</div>
|
||||
<div class='leftSepa textTopHeader'>
|
||||
<strong>Date:
|
||||
<input type="text" id="datepickerStart" size="30" style="margin-bottom: 5px;">
|
||||
<strong> to </strong>
|
||||
<input type="text" id="datepickerEnd" size="30" style="margin-bottom: 5px;">
|
||||
</strong>
|
||||
</div>
|
||||
</div>
|
||||
<!-- /.navbar-header -->
|
||||
|
||||
</nav>
|
||||
|
||||
<!-- Page Content -->
|
||||
<div id="page-wrapper" style="margin: 0px; padding: 0px;">
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
<div style="height: 10px;"></div>
|
||||
|
||||
<div class="col-lg-12">
|
||||
<div class="col-lg-4">
|
||||
<div class="panel panel-default" style="">
|
||||
<div class="panel-heading bg-info" style="font-weight: bold;">
|
||||
<b>Most active events</b>
|
||||
<input type="text" id="typeaheadEvent" data-provide="typeahead" size="20" style="margin-bottom: 5px; float:right;" placeholder="Enter an event">
|
||||
</div>
|
||||
<div id="panelbody" class="panel-body" style="">
|
||||
<div id="eventPie" style="width:100%; height: 34vh;"></div>
|
||||
<div id="eventLine" style="width:100%; height: 27vh;"></div>
|
||||
</div>
|
||||
</div><!-- /.panel-body -->
|
||||
</div>
|
||||
|
||||
<div class="col-lg-3">
|
||||
<div class="panel panel-default" style="">
|
||||
<div class="panel-heading bg-info" style="font-weight: bold;">
|
||||
<b>Most active categories</b>
|
||||
<input type="text" id="typeaheadCateg" data-provide="typeahead" size="20" style="margin-bottom: 5px; float:right;" placeholder="Enter a category">
|
||||
</div>
|
||||
<div id="panelbody" class="panel-body" style="">
|
||||
<div id="categPie" style="width:100%; height: 34vh;"></div>
|
||||
<div id="categLine" style="width:100%; height: 27vh;"></div>
|
||||
</div>
|
||||
</div><!-- /.panel-body -->
|
||||
</div>
|
||||
|
||||
<div class="col-lg-5">
|
||||
<div class="panel panel-default" style="">
|
||||
<div class="panel-heading bg-info" style="font-weight: bold;">
|
||||
<b>Most popular tags</b>
|
||||
<input type="text" id="typeaheadTag" data-provide="typeahead" size="20" style="margin-bottom: 5px; float:right;" placeholder="Enter a tag">
|
||||
</div>
|
||||
<div id="panelbody" class="panel-body" style="">
|
||||
<div id="tagPie" style="width:100%; height: 34vh;"></div>
|
||||
<div id="tagLine" style="width:100%; height: 27vh;"></div>
|
||||
</div>
|
||||
</div><!-- /.panel-body -->
|
||||
</div>
|
||||
|
||||
</div><!-- /.col-lg-12 -->
|
||||
<div class="col-lg-12">
|
||||
<div class="col-lg-6">
|
||||
<div class="panel panel-default" style="">
|
||||
<div class="panel-heading bg-info" style="font-weight: bold;">
|
||||
<b>Sightings</b>
|
||||
</div>
|
||||
<div id="panelbody" class="panel-body" style="">
|
||||
<div id="sightingLine" style="width:100%; height: 20vh;"></div>
|
||||
</div>
|
||||
</div><!-- /.panel-body -->
|
||||
</div>
|
||||
<div class="col-lg-6">
|
||||
<div class="panel panel-default" style="">
|
||||
<div class="panel-heading bg-info" style="font-weight: bold;">
|
||||
<b>Discussion</b>
|
||||
</div>
|
||||
<div id="panelbody" class="panel-body" style="">
|
||||
<div id="discussionLine" style="width:100%; height: 20vh;"></div>
|
||||
</div>
|
||||
</div><!-- /.panel-body -->
|
||||
</div>
|
||||
</div><!-- /.col-lg-12 -->
|
||||
|
||||
</div><!-- /.row -->
|
||||
|
||||
</div> <!-- /.container-fluid -->
|
||||
|
||||
</div>
|
||||
<!-- /#page-wrapper -->
|
||||
|
||||
</div>
|
||||
<!-- /#wrapper -->
|
||||
|
||||
<!-- Index -->
|
||||
<script>
|
||||
/* URL */
|
||||
var urlIconCalendar = "{{ url_for('static', filename='pics/calendar.gif') }}";
|
||||
var url_currentPage = "{{ url_for('trendings') }}";
|
||||
var url_getTrendingEvent = "{{ url_for('getTrendingEvents') }}";
|
||||
var url_getTrendingCateg = "{{ url_for('getTrendingCategs') }}";
|
||||
var url_getTrendingTag = "{{ url_for('getTrendingTags') }}";
|
||||
var url_getTrendingSightings = "{{ url_for('getTrendingSightings') }}";
|
||||
var url_getTrendingDisc = "{{ url_for('getTrendingDisc') }}";
|
||||
|
||||
var url_getTypeaheadData = "{{ url_for('getTypeaheadData') }}";
|
||||
|
||||
/* DATA FROM CONF */
|
||||
|
||||
</script>
|
||||
|
||||
<script src="{{ url_for('static', filename='js/trendings.js') }}"></script>
|
||||
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
|
@ -8,7 +8,7 @@
|
|||
<meta name="description" content="">
|
||||
<meta name="author" content="">
|
||||
|
||||
<title>MISP live dashboard</title>
|
||||
<title>MISP Users</title>
|
||||
|
||||
<!-- Bootstrap Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
|
||||
|
|
@ -109,7 +109,8 @@ small {
|
|||
<li><a href="{{ url_for('geo') }}">MISP Geolocalisation</a></li>
|
||||
<li><a href="{{ url_for('contrib') }}">MISP Contributors</a></li>
|
||||
<li><a href="{{ url_for('users') }}">MISP Users</a></li>
|
||||
</ul>
|
||||
<li><a href="{{ url_for('trendings') }}">MISP Trendings</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
<!-- /.navbar-header -->
|
||||
|
||||
|
|
@ -139,7 +140,7 @@ small {
|
|||
<div class="col-lg-5">
|
||||
<div class="panel panel-default" style="">
|
||||
<div class="panel-heading bg-info" style="font-weight: bold;">
|
||||
<b>Contribution/login (last 31 days)</b>
|
||||
<b>Contribution/login (last 7 days)</b>
|
||||
<strong class='leftSepa textTopHeader' style="float: none; padding: 11px;">Dates:
|
||||
<input type="text" id="datepickerOrgLogin" size="10" style="">
|
||||
</strong>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,139 @@
|
|||
import math, random
|
||||
import os
|
||||
import json
|
||||
import datetime, time
|
||||
from collections import OrderedDict
|
||||
|
||||
import util
|
||||
|
||||
class Trendings_helper:
|
||||
def __init__(self, serv_redis_db, cfg):
|
||||
self.serv_redis_db = serv_redis_db
|
||||
self.cfg = cfg
|
||||
|
||||
''' SETTER '''
|
||||
|
||||
def addGenericTrending(self, trendingType, data, timestamp):
|
||||
timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
|
||||
timestampDate_str = util.getDateStrFormat(timestampDate)
|
||||
keyname = "{}:{}".format(trendingType, timestampDate_str)
|
||||
if isinstance(data, OrderedDict):
|
||||
to_save = json.dumps(data)
|
||||
else:
|
||||
to_save = data
|
||||
self.serv_redis_db.zincrby(keyname, to_save, 1)
|
||||
|
||||
def addTrendingEvent(self, eventName, timestamp):
|
||||
self.addGenericTrending('TRENDINGS_EVENTS', eventName, timestamp)
|
||||
|
||||
def addTrendingCateg(self, categName, timestamp):
|
||||
self.addGenericTrending('TRENDINGS_CATEGS', categName, timestamp)
|
||||
|
||||
def addTrendingDisc(self, eventName, timestamp):
|
||||
self.addGenericTrending('TRENDINGS_DISC', eventName, timestamp)
|
||||
|
||||
def addTrendingTags(self, tags, timestamp):
|
||||
for tag in tags:
|
||||
ordDic = OrderedDict() #keep fields with the same layout in redis
|
||||
ordDic['id'] = tag['id']
|
||||
ordDic['name'] = tag['name']
|
||||
ordDic['colour'] = tag['colour']
|
||||
self.addGenericTrending('TRENDINGS_TAGS', ordDic, timestamp)
|
||||
|
||||
def addSightings(self, timestamp):
|
||||
timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
|
||||
timestampDate_str = util.getDateStrFormat(timestampDate)
|
||||
keyname = "{}:{}".format("TRENDINGS_SIGHT_sightings", timestampDate_str)
|
||||
self.serv_redis_db.incrby(keyname, 1)
|
||||
|
||||
def addFalsePositive(self, timestamp):
|
||||
timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
|
||||
timestampDate_str = util.getDateStrFormat(timestampDate)
|
||||
keyname = "{}:{}".format("TRENDINGS_SIGHT_false_positive", timestampDate_str)
|
||||
self.serv_redis_db.incrby(keyname, 1)
|
||||
|
||||
''' GETTER '''
|
||||
|
||||
def getGenericTrending(self, trendingType, dateS, dateE, topNum=0):
|
||||
to_ret = []
|
||||
prev_days = (dateE - dateS).days
|
||||
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
|
||||
keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate))
|
||||
data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True)
|
||||
data = [ [record[0].decode('utf8'), record[1]] for record in data ]
|
||||
data = data if data is not None else []
|
||||
to_ret.append([util.getTimestamp(curDate), data])
|
||||
return to_ret
|
||||
|
||||
def getSpecificTrending(self, trendingType, dateS, dateE, specificLabel=''):
|
||||
to_ret = []
|
||||
prev_days = (dateE - dateS).days
|
||||
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
|
||||
keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate))
|
||||
data = self.serv_redis_db.zscore(keyname, specificLabel)
|
||||
data = [[specificLabel, data]] if data is not None else []
|
||||
to_ret.append([util.getTimestamp(curDate), data])
|
||||
return to_ret
|
||||
|
||||
def getTrendingEvents(self, dateS, dateE, specificLabel=None):
|
||||
if specificLabel is None:
|
||||
return self.getGenericTrending('TRENDINGS_EVENTS', dateS, dateE)
|
||||
else:
|
||||
specificLabel = specificLabel.replace('\\n', '\n'); # reset correctly label with their \n (CR) instead of their char value
|
||||
return self.getSpecificTrending('TRENDINGS_EVENTS', dateS, dateE, specificLabel)
|
||||
|
||||
def getTrendingCategs(self, dateS, dateE):
|
||||
return self.getGenericTrending('TRENDINGS_CATEGS', dateS, dateE)
|
||||
|
||||
# FIXME: Construct this when getting data
|
||||
def getTrendingTags(self, dateS, dateE, topNum=12):
|
||||
to_ret = []
|
||||
prev_days = (dateE - dateS).days
|
||||
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
|
||||
keyname = "{}:{}".format('TRENDINGS_TAGS', util.getDateStrFormat(curDate))
|
||||
data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True)
|
||||
data = [ [record[0].decode('utf8'), record[1]] for record in data ]
|
||||
data = data if data is not None else []
|
||||
temp = []
|
||||
for jText, score in data:
|
||||
temp.append([json.loads(jText), score])
|
||||
data = temp
|
||||
to_ret.append([util.getTimestamp(curDate), data])
|
||||
return to_ret
|
||||
|
||||
def getTrendingSightings(self, dateS, dateE):
|
||||
to_ret = []
|
||||
prev_days = (dateE - dateS).days
|
||||
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
|
||||
keyname = "{}:{}".format("TRENDINGS_SIGHT_sightings", util.getDateStrFormat(curDate))
|
||||
sight = self.serv_redis_db.get(keyname)
|
||||
sight = 0 if sight is None else int(sight.decode('utf8'))
|
||||
keyname = "{}:{}".format("TRENDINGS_SIGHT_false_positive", util.getDateStrFormat(curDate))
|
||||
fp = self.serv_redis_db.get(keyname)
|
||||
fp = 0 if fp is None else int(fp.decode('utf8'))
|
||||
to_ret.append([util.getTimestamp(curDate), { 'sightings': sight, 'false_positive': fp}])
|
||||
return to_ret
|
||||
|
||||
def getTrendingDisc(self, dateS, dateE):
|
||||
return self.getGenericTrending('TRENDINGS_DISC', dateS, dateE)
|
||||
|
||||
def getTypeaheadData(self, dateS, dateE):
|
||||
to_ret = {}
|
||||
for trendingType in ['TRENDINGS_EVENTS', 'TRENDINGS_CATEGS']:
|
||||
allSet = set()
|
||||
prev_days = (dateE - dateS).days
|
||||
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
|
||||
keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate))
|
||||
data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True)
|
||||
for elem in data:
|
||||
allSet.add(elem.decode('utf8'))
|
||||
to_ret[trendingType] = list(allSet)
|
||||
tags = self.getTrendingTags(dateS, dateE)
|
||||
tagSet = set()
|
||||
for item in tags:
|
||||
theDate, tagList = item
|
||||
for tag in tagList:
|
||||
tag = tag[0]
|
||||
tagSet.add(tag['name'])
|
||||
to_ret['TRENDINGS_TAGS'] = list(tagSet)
|
||||
return to_ret
|
||||
|
|
@ -81,7 +81,7 @@ class Users_helper:
|
|||
totLog = 1
|
||||
return totContrib/totLog
|
||||
|
||||
def getTopOrglogin(self, date, maxNum=12, prev_days=31):
|
||||
def getTopOrglogin(self, date, maxNum=12, prev_days=7):
|
||||
all_logged_in_orgs = self.getAllLoggedInOrgs(date, prev_days)
|
||||
data = []
|
||||
for org in all_logged_in_orgs:
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ import geoip2.database
|
|||
import util
|
||||
import contributor_helper
|
||||
import users_helper
|
||||
import trendings_helper
|
||||
|
||||
configfile = os.path.join(os.environ['DASH_CONFIG'], 'config.cfg')
|
||||
cfg = configparser.ConfigParser()
|
||||
|
|
@ -55,6 +56,7 @@ serv_redis_db = redis.StrictRedis(
|
|||
|
||||
contributor_helper = contributor_helper.Contributor_helper(serv_redis_db, cfg)
|
||||
users_helper = users_helper.Users_helper(serv_redis_db, cfg)
|
||||
trendings_helper = trendings_helper.Trendings_helper(serv_redis_db, cfg)
|
||||
|
||||
reader = geoip2.database.Reader(PATH_TO_DB)
|
||||
|
||||
|
|
@ -193,15 +195,15 @@ def handler_keepalive(zmq_name, jsonevent):
|
|||
publish_log(zmq_name, 'Keepalive', to_push)
|
||||
|
||||
def handler_user(zmq_name, jsondata):
|
||||
action = jsondata['action']
|
||||
json_user = jsondata['User']
|
||||
userID = json_user['id']
|
||||
org = userID
|
||||
try: #only consider user login
|
||||
timestamp = json_user['current_login']
|
||||
except KeyError:
|
||||
return
|
||||
if timestamp != 0: # "invited_by": "xxxx" ???
|
||||
json_org = jsondata['Organisation']
|
||||
org = json_org['name']
|
||||
if action == 'login': #only consider user login
|
||||
timestamp = int(time.time())
|
||||
users_helper.add_user_login(timestamp, org)
|
||||
else:
|
||||
pass
|
||||
|
||||
def handler_conversation(zmq_name, jsonevent):
|
||||
try: #only consider POST, not THREAD
|
||||
|
|
@ -212,11 +214,15 @@ def handler_conversation(zmq_name, jsonevent):
|
|||
org = jsonpost['org_name']
|
||||
categ = None
|
||||
action = 'add'
|
||||
eventName = 'no name or id yet...'
|
||||
handleContribution(zmq_name, org,
|
||||
'Discussion',
|
||||
None,
|
||||
action,
|
||||
isLabeled=False)
|
||||
# add Discussion
|
||||
nowSec = int(time.time())
|
||||
trendings_helper.addTrendingDisc(eventName, nowSec)
|
||||
|
||||
def handler_object(zmq_name, jsondata):
|
||||
print('obj')
|
||||
|
|
@ -234,9 +240,33 @@ def handler_sighting(zmq_name, jsondata):
|
|||
handleContribution(zmq_name, org, 'Sighting', categ, action, pntMultiplier=2)
|
||||
handler_attribute(zmq_name, jsonsight, hasAlreadyBeenContributed=True)
|
||||
|
||||
try:
|
||||
timestamp = jsonsight['date_sighting']
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
if jsonsight['type'] == "0": # sightings
|
||||
trendings_helper.addSightings(timestamp)
|
||||
elif jsonsight['type'] == "1": # false positive
|
||||
trendings_helper.addFalsePositive(timestamp)
|
||||
|
||||
def handler_event(zmq_name, jsonobj):
|
||||
#fields: threat_level_id, id, info
|
||||
jsonevent = jsonobj['Event']
|
||||
|
||||
#Add trending
|
||||
eventName = jsonevent['info']
|
||||
timestamp = jsonevent['timestamp']
|
||||
trendings_helper.addTrendingEvent(eventName, timestamp)
|
||||
try:
|
||||
temp = jsonobj['EventTag']
|
||||
tags = []
|
||||
for tag in temp:
|
||||
tags.append(tag['Tag'])
|
||||
except KeyError:
|
||||
tags = []
|
||||
trendings_helper.addTrendingTags(tags, timestamp)
|
||||
|
||||
#redirect to handler_attribute
|
||||
if 'Attribute' in jsonevent:
|
||||
attributes = jsonevent['Attribute']
|
||||
|
|
@ -273,6 +303,22 @@ def handler_attribute(zmq_name, jsonobj, hasAlreadyBeenContributed=False):
|
|||
if 'Attribute' in jsonobj:
|
||||
jsonattr = jsonobj['Attribute']
|
||||
|
||||
#Add trending
|
||||
categName = jsonattr['category']
|
||||
try:
|
||||
timestamp = jsonattr['timestamp']
|
||||
except KeyError:
|
||||
timestamp = int(time.time())
|
||||
trendings_helper.addTrendingCateg(categName, timestamp)
|
||||
try:
|
||||
temp = jsonattr['Tag']
|
||||
tags = []
|
||||
for tag in temp:
|
||||
tags.append(tag['Tag'])
|
||||
except KeyError:
|
||||
tags = []
|
||||
trendings_helper.addTrendingTags(tags, timestamp)
|
||||
|
||||
to_push = []
|
||||
for field in json.loads(cfg.get('Log', 'fieldname_order')):
|
||||
if type(field) is list:
|
||||
|
|
@ -315,7 +361,10 @@ def process_log(zmq_name, event):
|
|||
topic, eventdata = event.split(' ', maxsplit=1)
|
||||
jsonevent = json.loads(eventdata)
|
||||
print(event)
|
||||
dico_action[topic](zmq_name, jsonevent)
|
||||
try:
|
||||
dico_action[topic](zmq_name, jsonevent)
|
||||
except KeyError as e:
|
||||
print(e)
|
||||
|
||||
|
||||
def main(zmqName):
|
||||
|
|
@ -340,7 +389,8 @@ dico_action = {
|
|||
"misp_json_sighting": handler_sighting,
|
||||
"misp_json_organisation": handler_log,
|
||||
"misp_json_user": handler_user,
|
||||
"misp_json_conversation": handler_conversation
|
||||
"misp_json_conversation": handler_conversation,
|
||||
"misp_json_object_reference": handler_log,
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue