diff --git a/README.md b/README.md index ad31a5e..8fd1f8d 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,38 @@ # MISP-Dashboard An experimental dashboard showing live data and statistics from the ZMQ of one or more MISP instances. + +# Installation +- Launch ```./install_dependencies.sh``` from the MISP-Dashboard directory +- Update the configuration file ```config.cfg``` so that it matches your system + - Fields that you may change: + - RedisGlobal -> host + - RedisGlobal -> port + - RedisGlobal -> zmq_url + - RedisGlobal -> misp_web_url + - RedisMap -> pathMaxMindDB + +# Starting the System +- Activate your virtualenv ```. ./DASHENV/bin/activate``` +- Listen to the MISP feed by starting the zmq_subscriber ```./zmq_subscriber.py``` +- Start the Flask server ```./server.py``` +- Access the interface at ```http://localhost:8001/``` + +# Features + ## Live Dashboard - Possibility to subscribe to multiple ZMQ feeds - Shows direct contribution made by organisations - Shows live resolvable posted locations -![MISP event view](./screenshots/dashboard-live.png) +![Dashboard live](./screenshots/dashboard-live.png) ## Geolocalisation Dashboard - Provides historical geolocalised information to support security teams, CSIRTs or SOC finding threats in their constituency - Possibility to get geospatial information from specific regions -![MISP event view](./screenshots/dashboard-geo.png) +![Dashbaord geo](./screenshots/dashboard-geo.png) ## Contributors Dashboard @@ -29,25 +48,25 @@ __Includes__: - Gamification of the platform: - Two different levels of ranking with unique icons - Exclusive obtainable badges for source code contributors and donator - -![Dashboard-contributor2](./screenshots/dashboard-contributors2.png) -![Dashboard-contributor3](./screenshots/dashboard-contributors3.png) +![Dashboard contributor](./screenshots/dashboard-contributors2.png) +![Dashboard contributor2](./screenshots/dashboard-contributors3.png) -# Installation -- Launch ```./install_dependencies.sh``` from the MISP-Dashboard directory -- Update the configuration file ```config.cfg``` so that it matches your system - - Fields that you may change: - - RedisGlobal -> host - - RedisGlobal -> port - - RedisGlobal -> zmq_url - - RedisGlobal -> misp_web_url - -# Starting the System -- Activate your virtualenv ```. ./DASHENV/bin/activate``` -- Listen to the MISP feed by starting the zmq_subscriber ```./zmq_subscriber.py``` -- Start the Flask server ```./server.py``` -- Access the interface at ```http://localhost:8001/``` +## Users Dashboard + +- Shows when and how the platform is used: + - Login punchcard and overtime + - Contribution vs login + +![Dashboard users](./screenshots/dashboard-users.png) + +## Trendings Dashboard + +- Provides real time information to support security teams, CSIRTs or SOC showing current threats and activity + - Shows most active events, categories and tags + - Shows sightings and discussion overtime + +![Dashboard users](./screenshots/dashboard-trendings.png) # zmq_subscriber options ```usage: zmq_subscriber.py [-h] [-n ZMQNAME] [-u ZMQURL] @@ -62,7 +81,7 @@ optional arguments: The URL to connect to ``` -## License +# License Images and logos are handmade for: - rankingMISPOrg/ - rankingMISPMonthly/ diff --git a/config/config.cfg.default b/config/config.cfg.default index f0c0bfb..cdb6eed 100644 --- a/config/config.cfg.default +++ b/config/config.cfg.default @@ -27,7 +27,7 @@ rankMultiplier = 2 categories_in_datatable = ["internal_reference", "targeting_data", "antivirus_detection", "payload_delivery", "artifacts_dropped", "payload_installation", "persistence_mechanism", "network_activity", "payload_type", "attribution", "external_analysis", "financial_fraud", "support_Tool", "social_network", "person", "other" ] default_pnts_per_contribution = 1 # array of the form [[category, pntsRcv], ...] -pnts_per_contribution = [["payload_delivery", 1], ["artifact_dropped", 1], ["network_activity", 1]] +pnts_per_contribution = [["payload_delivery", 1], ["artifacts_dropped", 1], ["network_activity", 1]] additional_help_text = ["Sightings multiplies earned points by 2", "Editing an attribute earns you the same as creating one"] [Log] diff --git a/install_dependencies.sh b/install_dependencies.sh index b8bfe64..8677b05 100755 --- a/install_dependencies.sh +++ b/install_dependencies.sh @@ -3,7 +3,7 @@ set -e set -x -sudo apt-get install python3-virtualenv -y +sudo apt-get install python3-virtualenv virtualenv screen redis-server -y if [ -z "$VIRTUAL_ENV" ]; then virtualenv -p python3 DASHENV @@ -102,4 +102,10 @@ wget https://cdn.datatables.net/${DATATABLE_VERSION}/js/dataTables.bootstrap.js git clone https://github.com/bassjobsen/Bootstrap-3-Typeahead.git temp/Bootstrap-3-Typeahead mv temp/Bootstrap-3-Typeahead/bootstrap3-typeahead.min.js ./static/js +#punchcard +git clone https://github.com/melenaos/jquery-punchcard.git temp/jquery-punchcard +mv temp/jquery-punchcard/src/punchcard.js ./static/js +mv temp/jquery-punchcard/src/punchcard.css ./static/css +wget https://momentjs.com/downloads/moment.js -O ./static/js/moment.js + rm -rf ./temp diff --git a/screenshots/dashboard-trendings.png b/screenshots/dashboard-trendings.png new file mode 100644 index 0000000..e8937e4 Binary files /dev/null and b/screenshots/dashboard-trendings.png differ diff --git a/screenshots/dashboard-users.png b/screenshots/dashboard-users.png new file mode 100644 index 0000000..0058be1 Binary files /dev/null and b/screenshots/dashboard-users.png differ diff --git a/server.py b/server.py index c83f43c..546031f 100755 --- a/server.py +++ b/server.py @@ -12,6 +12,7 @@ import os import util import contributor_helper import users_helper +import trendings_helper configfile = os.path.join(os.environ['DASH_CONFIG'], 'config.cfg') cfg = configparser.ConfigParser() @@ -34,6 +35,7 @@ serv_redis_db = redis.StrictRedis( contributor_helper = contributor_helper.Contributor_helper(serv_redis_db, cfg) users_helper = users_helper.Users_helper(serv_redis_db, cfg) +trendings_helper = trendings_helper.Trendings_helper(serv_redis_db, cfg) subscriber_log = redis_server_log.pubsub(ignore_subscribe_messages=True) subscriber_log.psubscribe(cfg.get('RedisLog', 'channel')) @@ -208,6 +210,18 @@ def users(): ) +@app.route("/trendings") +def trendings(): + maxNum = request.args.get('maxNum') + try: + maxNum = int(maxNum) + except: + maxNum = 15 + + return render_template('trendings.html', + maxNum=maxNum + ) + ''' INDEX ''' @app.route("/_logs") @@ -497,6 +511,82 @@ def getUserLoginsAndContribOvertime(): data = users_helper.getUserLoginsAndContribOvertime(date) return jsonify(data) +''' TRENDINGS ''' +@app.route("/_getTrendingEvents") +def getTrendingEvents(): + try: + dateS = datetime.datetime.fromtimestamp(float(request.args.get('dateS'))) + dateE = datetime.datetime.fromtimestamp(float(request.args.get('dateE'))) + except: + dateS = datetime.datetime.now() - datetime.timedelta(days=7) + dateE = datetime.datetime.now() + + specificLabel = request.args.get('specificLabel') + data = trendings_helper.getTrendingEvents(dateS, dateE, specificLabel) + return jsonify(data) + +@app.route("/_getTrendingCategs") +def getTrendingCategs(): + try: + dateS = datetime.datetime.fromtimestamp(float(request.args.get('dateS'))) + dateE = datetime.datetime.fromtimestamp(float(request.args.get('dateE'))) + except: + dateS = datetime.datetime.now() - datetime.timedelta(days=7) + dateE = datetime.datetime.now() + + + data = trendings_helper.getTrendingCategs(dateS, dateE) + return jsonify(data) + +@app.route("/_getTrendingTags") +def getTrendingTags(): + try: + dateS = datetime.datetime.fromtimestamp(float(request.args.get('dateS'))) + dateE = datetime.datetime.fromtimestamp(float(request.args.get('dateE'))) + except: + dateS = datetime.datetime.now() - datetime.timedelta(days=7) + dateE = datetime.datetime.now() + + + data = trendings_helper.getTrendingTags(dateS, dateE) + return jsonify(data) + +@app.route("/_getTrendingSightings") +def getTrendingSightings(): + try: + dateS = datetime.datetime.fromtimestamp(float(request.args.get('dateS'))) + dateE = datetime.datetime.fromtimestamp(float(request.args.get('dateE'))) + except: + dateS = datetime.datetime.now() - datetime.timedelta(days=7) + dateE = datetime.datetime.now() + + data = trendings_helper.getTrendingSightings(dateS, dateE) + return jsonify(data) + +@app.route("/_getTrendingDisc") +def getTrendingDisc(): + try: + dateS = datetime.datetime.fromtimestamp(float(request.args.get('dateS'))) + dateE = datetime.datetime.fromtimestamp(float(request.args.get('dateE'))) + except: + dateS = datetime.datetime.now() - datetime.timedelta(days=7) + dateE = datetime.datetime.now() + + + data = trendings_helper.getTrendingDisc(dateS, dateE) + return jsonify(data) + +@app.route("/_getTypeaheadData") +def getTypeaheadData(): + try: + dateS = datetime.datetime.fromtimestamp(float(request.args.get('dateS'))) + dateE = datetime.datetime.fromtimestamp(float(request.args.get('dateE'))) + except: + dateS = datetime.datetime.now() - datetime.timedelta(days=7) + dateE = datetime.datetime.now() + + data = trendings_helper.getTypeaheadData(dateS, dateE) + return jsonify(data) if __name__ == '__main__': app.run(host='localhost', port=8001, threaded=True) diff --git a/static/css/ranking.css b/static/css/ranking.css index ce76ca8..df7d62d 100644 --- a/static/css/ranking.css +++ b/static/css/ranking.css @@ -90,6 +90,11 @@ display: none; } +.table > thead > tr > th.centerCell { + text-align: center; + min-width: 45px; +} + .table > tbody > tr > td.centerCell { text-align: center; min-width: 45px; diff --git a/static/js/trendings.js b/static/js/trendings.js new file mode 100644 index 0000000..79741e9 --- /dev/null +++ b/static/js/trendings.js @@ -0,0 +1,441 @@ +/* VARS */ +var dateStart; +var dateEnd; +var eventPie = ["#eventPie"]; +var eventLine = ["#eventLine"]; +var categPie = ["#categPie"]; +var categLine = ["#categLine"]; +var tagPie = ["#tagPie"]; +var tagLine = ["#tagLine"]; +var sightingLineWidget; +var discLine = ["#discussionLine"]; +var allData; +var globalColorMapping = {}; + +/* OPTIONS */ +var datePickerOptions = { + showOn: "button", + maxDate: 0, + buttonImage: urlIconCalendar, + buttonImageOnly: true, + buttonText: "Select date", + showAnim: "slideDown", + onSelect: dateChanged +}; +var lineChartOption = { + lines: { + show: true, + }, + points: { show: true }, + xaxis: { + mode: "time", + minTickSize: [1, "day"], + }, + legend: { show: false }, + grid: { + hoverable: true + } +}; +var pieChartOption = { + series: { + pie: { + innerRadius: 0.2, + show: true, + radius: 100, + label: { + show: true, + radius: 6/10, + formatter: innerPieLabelFormatter, + } + } + }, + legend: { + show: true, + labelFormatter: legendFormatter + }, + grid: { + hoverable: true, + clickable: true + } +}; +var typeaheadOption_event = { + source: function (query, process) { + if (allData === undefined) { // caching + return $.getJSON(url_getTypeaheadData, function (data) { + allData = data; + return process(data.TRENDINGS_EVENTS); + }); + } else { + return process(allData.TRENDINGS_EVENTS); + } + }, + updater: function(theevent) { + updateLineForLabel(eventLine, theevent, undefined, url_getTrendingEvent); + } +} +var typeaheadOption_categ = { + source: function (query, process) { + if (allData === undefined) { // caching + return $.getJSON(url_getTypeaheadData, function (data) { + allData = data; + return process(data.TRENDINGS_CATEGS); + }); + } else { + return process(allData.TRENDINGS_CATEGS); + } + }, + updater: function(categ) { + updateLineForLabel(categLine, categ, undefined, url_getTrendingCateg); + } +} +var typeaheadOption_tag = { + source: function (query, process) { + if (allData === undefined) { // caching + return $.getJSON(url_getTypeaheadData, function (data) { + allData = data; + return process(data.TRENDINGS_TAGS); + }); + } else { + return process(allData.TRENDINGS_TAGS); + } + }, + updater: function(tag) { + updateLineForLabel(tagLine, tag, undefined, url_getTrendingTag); + } +} + +/* FUNCTIONS */ +function getColor(label) { + try { + return globalColorMapping[label]; + } catch(err) { + return undefined; + } + +} + +function innerPieLabelFormatter(label, series) { + var count = series.data[0][1]; + return '
' + + ' ' + count + '' + + '
'; +} + +function getTextColour(rgb) { + var r = parseInt('0x'+rgb.substring(0,2)); + var g = parseInt('0x'+rgb.substring(2,4)); + var b = parseInt('0x'+rgb.substring(4,6)); + var avg = ((2 * r) + b + (3 * g))/6; + if (avg < 128) { + return 'white'; + } else { + return 'black'; + } +} +function legendFormatter(label, series) { + try { + jsonLabel = JSON.parse(label); + var backgroundColor = jsonLabel.colour; + var color = getTextColour(backgroundColor.substring(1,6));; + var labelText = jsonLabel.name; + return '
' + + ' ' + labelText + '' + + '
'; + } catch(err) { + // removing unwanted " + var label = label.replace(/\\"/g, "").replace(/\"/g, ""); + // limiting size + if (label.length >= 40){ + labelLimited = label.substring(0, 40) + '[...]'; + } else { + labelLimited = label; + } + return '
' + + ' ' + labelLimited + + ''; + + '
'; + } +} + +function generateEmptyAndFillData(data, specificLabel, colorMapping) { + // formating - Generate empty data + var toPlot_obj = {}; + var allDates = []; + for (var arr of data) { + var date = new Date(arr[0]*1000); + date = new Date(date.valueOf() - date.getTimezoneOffset() * 60000); // center the data around the day + allDates.push(date); + var items = arr[1]; + if (items.length > 0) { + for(var item_arr of items) { + var count = item_arr[1]; + var itemStr = JSON.stringify(item_arr[0]); + if (specificLabel === undefined || specificLabel == item_arr[0]) { // no tag + if(toPlot_obj[itemStr] === undefined) + toPlot_obj[itemStr] = {}; + toPlot_obj[itemStr][date] = count; + } else if (specificLabel == item_arr[0].name) { // tag + if(toPlot_obj[itemStr] === undefined) + toPlot_obj[itemStr] = {}; + toPlot_obj[itemStr][date] = count; + } else if (specificLabel == itemStr.substring(1, itemStr.length-1)) { // tag from click (countain { and }, need to supress it) + if(toPlot_obj[itemStr] === undefined) + toPlot_obj[itemStr] = {}; + toPlot_obj[itemStr][date] = count; + } + } + } + } + toPlot = [] + for (var itemStr in toPlot_obj) { + if (toPlot_obj.hasOwnProperty(itemStr)) { + data_toPlot = [] + for (var curDate of allDates) { + if (toPlot_obj[itemStr].hasOwnProperty(curDate)) { + data_toPlot.push([curDate, toPlot_obj[itemStr][curDate]]) + } else { + data_toPlot.push([curDate, 0]) + } + } + if (colorMapping === undefined) { + //try to get color, else no color + var colorCode = getColor(itemStr); + if (!( colorCode === undefined)) { + toPlot.push({label: itemStr, data: data_toPlot, color: colorCode}) + } else { + toPlot.push({label: itemStr, data: data_toPlot}) + } + } else { + try { + var color = colorMapping[itemStr].colour; + toPlot.push({label: itemStr, data: data_toPlot, color: color}) + } catch(err) { + // ignore, only shows data displayed in the pie chart + } + } + } + } + return toPlot; +} + +function compareObj(a,b) { + if (a.data < b.data) + return -1; + if (a.data > b.data) + return 1; + return 0; +} +/* UPDATES */ + +// return the color maping: label->color +function updatePie(pie, line, data, url) { + var pieID = pie[0]; + var pieWidget = pie[1]; + var itemMapping = {}; + var colorMapping = {}; + if (data === undefined || data.length == 0 || (data[0] == 0 && data[1] == 0)) { + toPlot = [{ label: 'No data', data: 100 }]; + } else { + toPlot_obj = {} + for (var arr of data) { + var date = arr[0]; + var items = arr[1] + for(var item_arr of items) { + var itemStr = JSON.stringify(item_arr[0]); + itemMapping[itemStr] = item_arr[0]; + var count = item_arr[1]; + if(toPlot_obj[itemStr] === undefined) + toPlot_obj[itemStr] = 0; + toPlot_obj[itemStr] += count; + } + } + if (Object.keys(toPlot_obj).length == 0) { // no data + toPlot = [{ label: 'No data', data: 100 }]; + } else { + toPlot = []; + for (var itemStr in toPlot_obj) { + if (toPlot_obj.hasOwnProperty(itemStr)) { + var itemColor = itemMapping[itemStr].colour + colorMapping[itemStr] = itemColor; + toPlot.push({label: itemStr, data: toPlot_obj[itemStr], color: itemColor}) + } + } + } + toPlot.sort(compareObj).reverse(); + var maxNum = $('#num_selector').val(); + toPlot = toPlot.slice(0,maxNum); // take at max 12 elements + } + if (!(pieWidget === undefined)) { + pieWidget.setData(toPlot); + pieWidget.setupGrid(); + pieWidget.draw(); + // fill colorMapping + for (item of pieWidget.getData()) { + colorMapping[item.label] = {colour: item.color}; + } + } else { + pieWidget = $.plot(pieID, toPlot, pieChartOption); + pie.push(pieWidget); + // Hover + $(pieID).bind("plothover", function (event, pos, item) { + if (item) { + $("#tooltip").html(legendFormatter(item.series.label)) + .css({top: pos.pageY+5, left: pos.pageX+5}) + .fadeIn(200); + } else { + $("#tooltip").hide(); + } + }); + // Click + $(pieID).bind("plotclick", function(event, pos, obj) { + if (!obj) { return; } + var specificLabel = obj.series.label; + colorMapping[specificLabel] = {}; + colorMapping[specificLabel] = { colour: obj.series.color }; + updateLineForLabel(line, specificLabel.substring(1, specificLabel.length-1), colorMapping, url); + }); + for (item of pieWidget.getData()) { + colorMapping[item.label] = {colour: item.color}; + } + } + return colorMapping; +} + +function updateLine(line, data, chartOptions, specificLabel, colorMapping) { + lineID = line[0]; + lineWidget = line[1]; + toPlot = generateEmptyAndFillData(data, specificLabel, colorMapping); + // plot + if (!(lineWidget === undefined)) { + lineWidget.setData(toPlot); + lineWidget.setupGrid(); + lineWidget.draw(); + } else { + if (chartOptions === undefined) { + chartOptions = lineChartOption; + } + lineWidget = $.plot(lineID, toPlot, chartOptions); + line.push(lineWidget); + $(lineID).bind("plothover", function (event, pos, item) { + if (item) { + $("#tooltip").html(legendFormatter(item.series.label)) + .css({top: item.pageY+5, left: item.pageX+5}) + .fadeIn(200); + } else { + $("#tooltip").hide(); + } + }); + } +} + +function updateSignthingsChart() { + $.getJSON( url_getTrendingSightings+"?dateS="+parseInt(dateStart.getTime()/1000)+"&dateE="+parseInt(dateEnd.getTime()/1000), function( data ) { + var toPlot_obj = {}; + toPlot_obj['Sightings'] = []; + toPlot_obj['False positive'] = []; + var allDates = []; + for (var arr of data) { + var date = new Date(arr[0]*1000); + date = new Date(date.valueOf() - date.getTimezoneOffset() * 60000); // center the data around the day + allDates.push(date); + var items = arr[1]; + var sight = items.sightings; + var fp = items.false_positive; + toPlot_obj['Sightings'].push([date, sight]); + toPlot_obj['False positive'].push([date, -fp]); + } + toPlot = [] + toPlot.push({label: 'Sightings', data: toPlot_obj['Sightings'], color: '#4da74d'}) + toPlot.push({label: 'False positive', data: toPlot_obj['False positive'], color: '#cb4b4b'}) + + if (!(sightingLineWidget === undefined)) { + sightingLineWidget.setData(toPlot); + sightingLineWidget.setupGrid(); + sightingLineWidget.draw(); + } else { + var lineChartOptionSight = jQuery.extend(true, {}, lineChartOption); + lineChartOptionSight['legend']['show'] = true; + lineChartOptionSight['legend']['position'] = 'nw'; + lineChartOptionSight['grid'] = {}; + lineChartOptionSight['lines']['fill'] = true; + sightingLineWidget = $.plot("#sightingLine", toPlot, lineChartOptionSight); + } + }); +} + +function updateLineForLabel(line, specificLabel, colorMapping, url) { + $.getJSON( url+"?dateS="+parseInt(dateStart.getTime()/1000)+"&dateE="+parseInt(dateEnd.getTime()/1000)+"&specificLabel="+specificLabel, function( data ) { + updateLine(line, data, undefined, specificLabel, colorMapping); + }); +} + +function updatePieLine(pie, line, url) { + $.getJSON( url+"?dateS="+parseInt(dateStart.getTime()/1000)+"&dateE="+parseInt(dateEnd.getTime()/1000), function( data ) { + var colorMapping = updatePie(pie, line, data, url); + for (var item in colorMapping) { + if (colorMapping.hasOwnProperty(item) && colorMapping[item] != undefined) { + globalColorMapping[item] = colorMapping[item].colour; + } + } + updateLine(line, data, undefined, undefined, colorMapping); + }); +} + +function updateDisc() { + var lineChartOptionDisc = jQuery.extend(true, {}, lineChartOption); + lineChartOptionDisc['legend']['show'] = true; + lineChartOptionDisc['legend']['position'] = 'nw'; + lineChartOptionDisc['lines']['fill'] = true; + $.getJSON( url_getTrendingDisc+"?dateS="+parseInt(dateStart.getTime()/1000)+"&dateE="+parseInt(dateEnd.getTime()/1000), function( data ) { + updateLine(discLine, data, lineChartOptionDisc); + }); +} + +function dateChanged() { + dateStart = datePickerWidgetStart.datepicker( "getDate" ); + dateEnd = datePickerWidgetEnd.datepicker( "getDate" ); + updatePieLine(eventPie, eventLine, url_getTrendingEvent); + updatePieLine(categPie, categLine, url_getTrendingCateg); + updatePieLine(tagPie, tagLine, url_getTrendingTag); + updateSignthingsChart(); + updateDisc(); +} + +$(document).ready(function () { + + datePickerWidgetStart = $( "#datepickerStart" ).datepicker(datePickerOptions); + var lastWeekDate = new Date(); + lastWeekDate.setDate(lastWeekDate.getDate()-7); + datePickerWidgetStart.datepicker("setDate", lastWeekDate); + dateStart = datePickerWidgetStart.datepicker( "getDate" ); + datePickerWidgetEnd = $( "#datepickerEnd" ).datepicker(datePickerOptions); + datePickerWidgetEnd.datepicker("setDate", new Date()); + dateEnd = datePickerWidgetEnd.datepicker( "getDate" ); + + $('#typeaheadEvent').typeahead(typeaheadOption_event); + $('#typeaheadCateg').typeahead(typeaheadOption_categ); + $('#typeaheadTag').typeahead(typeaheadOption_tag); + + updatePieLine(eventPie, eventLine, url_getTrendingEvent) + updatePieLine(categPie, categLine, url_getTrendingCateg) + updatePieLine(tagPie, tagLine, url_getTrendingTag) + updateSignthingsChart(); + updateDisc(); + + $( "#num_selector" ).change(function() { + var sel = parseInt($( this ).val()); + var maxNum = sel; + window.location.href = url_currentPage+'?maxNum='+maxNum; + }); + + $("
").css({ + position: "absolute", + display: "none", + }).appendTo("body"); + +}); diff --git a/templates/contrib.html b/templates/contrib.html index 43ec8b6..4faee41 100644 --- a/templates/contrib.html +++ b/templates/contrib.html @@ -8,7 +8,7 @@ - MISP live dashboard + MISP Contributors @@ -188,14 +188,14 @@ {% for title in trophy_title_str %} - {{ title }} + {{ title }} {% endfor %} {% for perc in trophy_mapping %} - {{ perc }} + {{ perc }} {% endfor %} @@ -249,6 +249,8 @@
  • MISP Live Dashboard
  • MISP Geolocalisation
  • MISP Contributors
    • +
  • MISP Users
    • +
  • MISP Trendings
    • diff --git a/templates/geo.html b/templates/geo.html index 2dfbfde..1579801 100644 --- a/templates/geo.html +++ b/templates/geo.html @@ -8,7 +8,7 @@ - MISP live dashboard + MISP Geolocalisation @@ -91,6 +91,8 @@ small {
  • MISP Live Dashboard
  • MISP Geolocalisation
  • MISP Contributors
    • +
  • MISP Users
    • +
  • MISP Trendings
    • diff --git a/templates/index.html b/templates/index.html index 63b6f5a..ae1a69c 100644 --- a/templates/index.html +++ b/templates/index.html @@ -133,6 +133,8 @@ small {
  • MISP Live Dashboard
  • MISP Geolocalisation
  • MISP Contributors
    • +
  • MISP Users
    • +
  • MISP Trendings
    • diff --git a/templates/trendings.html b/templates/trendings.html new file mode 100644 index 0000000..924e316 --- /dev/null +++ b/templates/trendings.html @@ -0,0 +1,240 @@ + + + + + + + + + + + MISP Trendings + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + + + + + +
    +
    +
    +
    + +
    +
    +
    +
    + Most active events + +
    +
    +
    +
    +
    +
    +
    + +
    +
    +
    + Most active categories + +
    +
    +
    +
    +
    +
    +
    + +
    +
    +
    + Most popular tags + +
    +
    +
    +
    +
    +
    +
    + +
    +
    +
    +
    +
    + Sightings +
    +
    +
    +
    +
    +
    +
    +
    +
    + Discussion +
    +
    +
    +
    +
    +
    +
    + +
    + +
    + +
    + + +
    + + + + + + + + + + diff --git a/templates/users.html b/templates/users.html index 5b59634..224b5b0 100644 --- a/templates/users.html +++ b/templates/users.html @@ -8,7 +8,7 @@ - MISP live dashboard + MISP Users @@ -109,7 +109,8 @@ small {
  • MISP Geolocalisation
  • MISP Contributors
  • MISP Users
    • - +
  • MISP Trendings
    • +
    @@ -139,7 +140,7 @@ small {
    - Contribution/login (last 31 days) + Contribution/login (last 7 days) Dates: diff --git a/trendings_helper.py b/trendings_helper.py new file mode 100644 index 0000000..cd04010 --- /dev/null +++ b/trendings_helper.py @@ -0,0 +1,139 @@ +import math, random +import os +import json +import datetime, time +from collections import OrderedDict + +import util + +class Trendings_helper: + def __init__(self, serv_redis_db, cfg): + self.serv_redis_db = serv_redis_db + self.cfg = cfg + + ''' SETTER ''' + + def addGenericTrending(self, trendingType, data, timestamp): + timestampDate = datetime.datetime.fromtimestamp(float(timestamp)) + timestampDate_str = util.getDateStrFormat(timestampDate) + keyname = "{}:{}".format(trendingType, timestampDate_str) + if isinstance(data, OrderedDict): + to_save = json.dumps(data) + else: + to_save = data + self.serv_redis_db.zincrby(keyname, to_save, 1) + + def addTrendingEvent(self, eventName, timestamp): + self.addGenericTrending('TRENDINGS_EVENTS', eventName, timestamp) + + def addTrendingCateg(self, categName, timestamp): + self.addGenericTrending('TRENDINGS_CATEGS', categName, timestamp) + + def addTrendingDisc(self, eventName, timestamp): + self.addGenericTrending('TRENDINGS_DISC', eventName, timestamp) + + def addTrendingTags(self, tags, timestamp): + for tag in tags: + ordDic = OrderedDict() #keep fields with the same layout in redis + ordDic['id'] = tag['id'] + ordDic['name'] = tag['name'] + ordDic['colour'] = tag['colour'] + self.addGenericTrending('TRENDINGS_TAGS', ordDic, timestamp) + + def addSightings(self, timestamp): + timestampDate = datetime.datetime.fromtimestamp(float(timestamp)) + timestampDate_str = util.getDateStrFormat(timestampDate) + keyname = "{}:{}".format("TRENDINGS_SIGHT_sightings", timestampDate_str) + self.serv_redis_db.incrby(keyname, 1) + + def addFalsePositive(self, timestamp): + timestampDate = datetime.datetime.fromtimestamp(float(timestamp)) + timestampDate_str = util.getDateStrFormat(timestampDate) + keyname = "{}:{}".format("TRENDINGS_SIGHT_false_positive", timestampDate_str) + self.serv_redis_db.incrby(keyname, 1) + + ''' GETTER ''' + + def getGenericTrending(self, trendingType, dateS, dateE, topNum=0): + to_ret = [] + prev_days = (dateE - dateS).days + for curDate in util.getXPrevDaysSpan(dateE, prev_days): + keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate)) + data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True) + data = [ [record[0].decode('utf8'), record[1]] for record in data ] + data = data if data is not None else [] + to_ret.append([util.getTimestamp(curDate), data]) + return to_ret + + def getSpecificTrending(self, trendingType, dateS, dateE, specificLabel=''): + to_ret = [] + prev_days = (dateE - dateS).days + for curDate in util.getXPrevDaysSpan(dateE, prev_days): + keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate)) + data = self.serv_redis_db.zscore(keyname, specificLabel) + data = [[specificLabel, data]] if data is not None else [] + to_ret.append([util.getTimestamp(curDate), data]) + return to_ret + + def getTrendingEvents(self, dateS, dateE, specificLabel=None): + if specificLabel is None: + return self.getGenericTrending('TRENDINGS_EVENTS', dateS, dateE) + else: + specificLabel = specificLabel.replace('\\n', '\n'); # reset correctly label with their \n (CR) instead of their char value + return self.getSpecificTrending('TRENDINGS_EVENTS', dateS, dateE, specificLabel) + + def getTrendingCategs(self, dateS, dateE): + return self.getGenericTrending('TRENDINGS_CATEGS', dateS, dateE) + + # FIXME: Construct this when getting data + def getTrendingTags(self, dateS, dateE, topNum=12): + to_ret = [] + prev_days = (dateE - dateS).days + for curDate in util.getXPrevDaysSpan(dateE, prev_days): + keyname = "{}:{}".format('TRENDINGS_TAGS', util.getDateStrFormat(curDate)) + data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True) + data = [ [record[0].decode('utf8'), record[1]] for record in data ] + data = data if data is not None else [] + temp = [] + for jText, score in data: + temp.append([json.loads(jText), score]) + data = temp + to_ret.append([util.getTimestamp(curDate), data]) + return to_ret + + def getTrendingSightings(self, dateS, dateE): + to_ret = [] + prev_days = (dateE - dateS).days + for curDate in util.getXPrevDaysSpan(dateE, prev_days): + keyname = "{}:{}".format("TRENDINGS_SIGHT_sightings", util.getDateStrFormat(curDate)) + sight = self.serv_redis_db.get(keyname) + sight = 0 if sight is None else int(sight.decode('utf8')) + keyname = "{}:{}".format("TRENDINGS_SIGHT_false_positive", util.getDateStrFormat(curDate)) + fp = self.serv_redis_db.get(keyname) + fp = 0 if fp is None else int(fp.decode('utf8')) + to_ret.append([util.getTimestamp(curDate), { 'sightings': sight, 'false_positive': fp}]) + return to_ret + + def getTrendingDisc(self, dateS, dateE): + return self.getGenericTrending('TRENDINGS_DISC', dateS, dateE) + + def getTypeaheadData(self, dateS, dateE): + to_ret = {} + for trendingType in ['TRENDINGS_EVENTS', 'TRENDINGS_CATEGS']: + allSet = set() + prev_days = (dateE - dateS).days + for curDate in util.getXPrevDaysSpan(dateE, prev_days): + keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate)) + data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True) + for elem in data: + allSet.add(elem.decode('utf8')) + to_ret[trendingType] = list(allSet) + tags = self.getTrendingTags(dateS, dateE) + tagSet = set() + for item in tags: + theDate, tagList = item + for tag in tagList: + tag = tag[0] + tagSet.add(tag['name']) + to_ret['TRENDINGS_TAGS'] = list(tagSet) + return to_ret diff --git a/users_helper.py b/users_helper.py index e70cd5e..48aed17 100644 --- a/users_helper.py +++ b/users_helper.py @@ -81,7 +81,7 @@ class Users_helper: totLog = 1 return totContrib/totLog - def getTopOrglogin(self, date, maxNum=12, prev_days=31): + def getTopOrglogin(self, date, maxNum=12, prev_days=7): all_logged_in_orgs = self.getAllLoggedInOrgs(date, prev_days) data = [] for org in all_logged_in_orgs: diff --git a/zmq_subscriber.py b/zmq_subscriber.py index 458c4a5..39fae6f 100755 --- a/zmq_subscriber.py +++ b/zmq_subscriber.py @@ -17,6 +17,7 @@ import geoip2.database import util import contributor_helper import users_helper +import trendings_helper configfile = os.path.join(os.environ['DASH_CONFIG'], 'config.cfg') cfg = configparser.ConfigParser() @@ -55,6 +56,7 @@ serv_redis_db = redis.StrictRedis( contributor_helper = contributor_helper.Contributor_helper(serv_redis_db, cfg) users_helper = users_helper.Users_helper(serv_redis_db, cfg) +trendings_helper = trendings_helper.Trendings_helper(serv_redis_db, cfg) reader = geoip2.database.Reader(PATH_TO_DB) @@ -193,15 +195,15 @@ def handler_keepalive(zmq_name, jsonevent): publish_log(zmq_name, 'Keepalive', to_push) def handler_user(zmq_name, jsondata): + action = jsondata['action'] json_user = jsondata['User'] - userID = json_user['id'] - org = userID - try: #only consider user login - timestamp = json_user['current_login'] - except KeyError: - return - if timestamp != 0: # "invited_by": "xxxx" ??? + json_org = jsondata['Organisation'] + org = json_org['name'] + if action == 'login': #only consider user login + timestamp = int(time.time()) users_helper.add_user_login(timestamp, org) + else: + pass def handler_conversation(zmq_name, jsonevent): try: #only consider POST, not THREAD @@ -212,11 +214,15 @@ def handler_conversation(zmq_name, jsonevent): org = jsonpost['org_name'] categ = None action = 'add' + eventName = 'no name or id yet...' handleContribution(zmq_name, org, 'Discussion', None, action, isLabeled=False) + # add Discussion + nowSec = int(time.time()) + trendings_helper.addTrendingDisc(eventName, nowSec) def handler_object(zmq_name, jsondata): print('obj') @@ -234,9 +240,33 @@ def handler_sighting(zmq_name, jsondata): handleContribution(zmq_name, org, 'Sighting', categ, action, pntMultiplier=2) handler_attribute(zmq_name, jsonsight, hasAlreadyBeenContributed=True) + try: + timestamp = jsonsight['date_sighting'] + except KeyError: + pass + + if jsonsight['type'] == "0": # sightings + trendings_helper.addSightings(timestamp) + elif jsonsight['type'] == "1": # false positive + trendings_helper.addFalsePositive(timestamp) + def handler_event(zmq_name, jsonobj): #fields: threat_level_id, id, info jsonevent = jsonobj['Event'] + + #Add trending + eventName = jsonevent['info'] + timestamp = jsonevent['timestamp'] + trendings_helper.addTrendingEvent(eventName, timestamp) + try: + temp = jsonobj['EventTag'] + tags = [] + for tag in temp: + tags.append(tag['Tag']) + except KeyError: + tags = [] + trendings_helper.addTrendingTags(tags, timestamp) + #redirect to handler_attribute if 'Attribute' in jsonevent: attributes = jsonevent['Attribute'] @@ -273,6 +303,22 @@ def handler_attribute(zmq_name, jsonobj, hasAlreadyBeenContributed=False): if 'Attribute' in jsonobj: jsonattr = jsonobj['Attribute'] + #Add trending + categName = jsonattr['category'] + try: + timestamp = jsonattr['timestamp'] + except KeyError: + timestamp = int(time.time()) + trendings_helper.addTrendingCateg(categName, timestamp) + try: + temp = jsonattr['Tag'] + tags = [] + for tag in temp: + tags.append(tag['Tag']) + except KeyError: + tags = [] + trendings_helper.addTrendingTags(tags, timestamp) + to_push = [] for field in json.loads(cfg.get('Log', 'fieldname_order')): if type(field) is list: @@ -315,7 +361,10 @@ def process_log(zmq_name, event): topic, eventdata = event.split(' ', maxsplit=1) jsonevent = json.loads(eventdata) print(event) - dico_action[topic](zmq_name, jsonevent) + try: + dico_action[topic](zmq_name, jsonevent) + except KeyError as e: + print(e) def main(zmqName): @@ -340,7 +389,8 @@ dico_action = { "misp_json_sighting": handler_sighting, "misp_json_organisation": handler_log, "misp_json_user": handler_user, - "misp_json_conversation": handler_conversation + "misp_json_conversation": handler_conversation, + "misp_json_object_reference": handler_log, }