Added support of named ZMQ + automatic Led colors in dashboard

server.py

@@ -78,11 +78,12 @@ class EventMessage():
             jsonMsg = { 'name': "undefined" ,'log': json.loads(msg) }
 
         self.feedName = jsonMsg['name']
+        self.zmqName = jsonMsg['zmqName']
         self.feed = json.loads(jsonMsg['log'])
         self.feed = LogItem(self.feed).get_row()
 
     def to_json(self):
-        to_ret = { 'log': self.feed, 'feedName': self.feedName }
+        to_ret = { 'log': self.feed, 'feedName': self.feedName, 'zmqName': self.zmqName }
         return 'data: {}\n\n'.format(json.dumps(to_ret))
 
 @app.route("/")

static/js/index/index.js

@@ -7,6 +7,68 @@ for(i=0; i<maxNumPoint; i++) {
     emptyArray.push([i, 0]);
 }
 
+class LedManager {
+    constructor() {
+        this._feedLedsTimeout = setTimeout(function(){ this.manageColors(); }, feedStatusFreqCheck);
+        this._feedLedKeepAlive = {};
+        this._allFeedName = [];
+        this._ledNum = 0;
+        this._nameToNumMapping = {}; //avoid bad ID if zmqname contains spaces
+    }
+
+    add_new_led(zmqname) {
+        this._allFeedName.push(zmqname);
+        this._nameToNumMapping[zmqname] = this._ledNum;
+        this._ledNum += 1;
+        this.add_new_html_led(zmqname);
+        this._feedLedKeepAlive[zmqname] = new Date().getTime();
+    }
+
+    add_new_html_led(zmqname) {
+        var ID = this._nameToNumMapping[zmqname]
+        var text = document.createElement('b');
+        text.innerHTML = zmqname;
+        var div = document.createElement('DIV');
+        div.id = "status_led_"+ID;
+        div.classList.add("led_green");
+        var sepa = document.createElement('DIV');
+        sepa.classList.add("leftSepa");
+        sepa.classList.add("textTopHeader");
+        sepa.appendChild(text);
+        sepa.appendChild(div);
+        $('#ledsHolder').append(sepa);
+    }
+
+    updateKeepAlive(zmqname) {
+        if (this._allFeedName.indexOf(zmqname) == -1) {
+            this.add_new_led(zmqname);
+        }
+        this._feedLedKeepAlive[zmqname] = new Date().getTime();
+        this.resetTimeoutAndRestart(zmqname);
+    }
+
+    resetTimeoutAndRestart(zmqName) {
+        clearTimeout(this._feedLedsTimeout); //cancel current leds timeout
+        this.manageColors();
+    }
+
+    manageColors() {
+        for (var feed in this._feedLedKeepAlive) {
+            var feedID = this._nameToNumMapping[feed];
+            var htmlLed = $("#status_led_"+feedID);
+            if(new Date().getTime() - this._feedLedKeepAlive[feed] > feedStatusFreqCheck) { // no feed
+                htmlLed.removeClass("led_green");
+                htmlLed.addClass("led_red");
+            } else {
+                htmlLed.removeClass("led_red");
+                htmlLed.addClass("led_green");
+            }
+        }
+        this._feedLedsTimeout = setTimeout(function(){ ledmanager.manageColors(); }, feedStatusFreqCheck);
+    }
+
+}
+
 class Sources {
     constructor() {
         this._sourcesArray = {};
@@ -101,6 +163,7 @@ class Sources {
 
 var sources = new Sources();
 sources.addSource('global');
+var ledmanager = new LedManager();
 
 var curNumLog = 0;
 var curMaxDataNumLog = 0;
@@ -120,7 +183,7 @@ function connect_source_log() {
 
     source_log.onmessage = function(event) {
         var json = jQuery.parseJSON( event.data );
-        updateLogTable(json.feedName, json.log);
+        updateLogTable(json.feedName, json.log, json.zmqName);
     };
 }
 
@@ -156,25 +219,15 @@ $(document).ready(function () {
 
 });
 
-function ledColorManager() {
-    $("#status_led").removeClass("led_orange"); 
-    if(new Date().getTime() - keepaliveTime > feedStatusFreqCheck) { // no feed
-        $("#status_led").removeClass("led_green");
-        $("#status_led").addClass("led_red");
-    } else {
-        $("#status_led").removeClass("led_red");
-        $("#status_led").addClass("led_green");
-    }
-    _timeoutLed = setTimeout(function(){ ledColorManager(); }, feedStatusFreqCheck);
-}
-_timeoutLed = setTimeout(function(){ ledColorManager(); }, feedStatusFreqCheck);
-
 
 //  LOG TABLE
-function updateLogTable(feedName, log) {
+function updateLogTable(feedName, log, zmqName) {
     if (log.length == 0)
         return;
 
+    // update keepAlives
+    ledmanager.updateKeepAlive(zmqName);
+
     // Create new row
     tableBody = document.getElementById('table_log_body');
 
@@ -200,13 +253,11 @@ function updateLogTable(feedName, log) {
             }
         }
     } else if (feedName == "Keepalive") {
-        keepaliveTime = new Date().getTime();
+        // do nothing
-        clearTimeout(_timeoutLed); //cancel current led timeout
-        ledColorManager();
     } else {
         // do nothing
-        return;
     }
+
 }
 
 function slideAndMax(orig, newData) {

templates/index.html

@@ -30,36 +30,33 @@ table {
 }
 
 .led_green {
-    float: right;
     margin: auto auto;
-    margin-top: 12.5px;
+    margin-top: 7.5px;
-    width: 24px;
+    width: 15px;
-    height: 24px;
+    height: 15px;
     background-color: #ABFF00;
     border-radius: 50%;
-    box-shadow: rgba(0, 0, 0, 0.2) 0 -1px 7px 1px, inset #304701 0 -1px 9px, #89FF00 0 2px 12px;
+    box-shadow: rgba(0, 0, 0, 0.2) 0 -1px 7px 1px, inset #304701 0 -1px 9px, #89FF00 0 2px 5px;
 }
 
 .led_red {
-    float: right;
     margin: auto auto;
-    margin-top: 12.5px;
+    margin-top: 7.5px;
-    width: 24px;
+    width: 15px;
-    height: 24px;
+    height: 15px;
     background-color: #F82222;
     border-radius: 50%;
-    box-shadow: rgba(0, 0, 0, 0.2) 0 -1px 7px 1px, inset #304701 0 -1px 9px, #FF0303 0 2px 12px;
+    box-shadow: rgba(0, 0, 0, 0.2) 0 -1px 7px 1px, inset #304701 0 -1px 9px, #FF0303 0 2px 5px;
 }
 
 .led_orange {
-    float: right;
     margin: auto auto;
-    margin-top: 12.5px;
+    margin-top: 7.5px;
-    width: 24px;
+    width: 15px;
-    height: 24px;
+    height: 15px;
     background-color: #FFB400;
     border-radius: 50%;
-    box-shadow: rgba(0, 0, 0, 0.2) 0 -1px 7px 1px, inset #304701 0 -1px 9px, #FF9000 0 2px 12px;
+    box-shadow: rgba(0, 0, 0, 0.2) 0 -1px 7px 1px, inset #304701 0 -1px 9px, #FF9000 0 2px 5px;
 }
 
 .marker_animation {
@@ -93,6 +90,13 @@ table {
     padding-left: 8px;
 }
 
+.textTopHeader {
+    height: 50px; 
+    text-align: center;
+    margin-left: 8px;
+    float: left;
+}
+
 small {
     font-size: 100%;
     font-weight: bold;
@@ -108,7 +112,7 @@ small {
         <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0; padding-left: 15px;">
             <div class="navbar-header">
                 <a class="navbar-brand" href="{{ url_for('index') }}">Misp feed dashboard</a>
-                <div id="status_led" class="led_orange"></div>
+                <div id="ledsHolder" style="float: right; height: 50px;"></div>
             </div>
             <!-- /.navbar-header -->
 

zmq_subscriber.py

@@ -5,6 +5,7 @@ import zmq
 import redis
 import random
 import configparser
+import argparse
 import os
 import sys
 import json
@@ -35,8 +36,8 @@ reader = geoip2.database.Reader(path_to_db)
 
 channel_proc = "CoordToProcess"
 
-def publish_log(name, content):
+def publish_log(zmq_name, name, content):
-    to_send = { 'name': name, 'log': json.dumps(content) }
+    to_send = { 'name': name, 'log': json.dumps(content), 'zmqName': zmq_name }
     redis_server.publish(channel, json.dumps(to_send))
 
 
@@ -50,7 +51,7 @@ def ip_to_coord(ip):
     lon_corrected = float("{:.4f}".format(lon))
     return { 'coord': {'lat': lat_corrected, 'lon': lon_corrected}, 'full_rep': resp }
 
-def getCoordAndPublish(supposed_ip, categ):
+def getCoordAndPublish(zmq_name, supposed_ip, categ):
     try:
         rep = ip_to_coord(supposed_ip)
         coord = rep['coord']
@@ -77,17 +78,17 @@ def getCoordAndPublish(supposed_ip, categ):
 ## HANDLERS ##
 ##############
 
-def handler_log(jsonevent):
+def handler_log(zmq_name, jsonevent):
     print('sending', 'log')
     return
 
-def handler_keepalive(jsonevent):
+def handler_keepalive(zmq_name, jsonevent):
     print('sending', 'keepalive')
     to_push = [ jsonevent['uptime'] ]
-    publish_log('Keepalive', to_push)
+    publish_log(zmq_name, 'Keepalive', to_push)
 
-def handler_event(jsonevent):
+def handler_event(zmq_name, jsonevent):
-    print(jsonevent)
+    #print(jsonevent)
     #fields: threat_level_id, id, info
     jsonevent = jsonevent['Event']
     #redirect to handler_attribute
@@ -95,16 +96,17 @@ def handler_event(jsonevent):
         attributes = jsonevent['Attribute']
         if attributes is list:
             for attr in attributes:
-                handler_attribute(attr)
+                handler_attribute(zmq_name, attr)
         else:
-            handler_attribute(jsonevent)
+            handler_attribute(zmq_name, jsonevent)
 
 
-def handler_attribute(jsonattr):
+def handler_attribute(zmq_name, jsonattr):
-    print(jsonattr)
     jsonattr = jsonattr['Attribute']
+    print(jsonattr)
     to_push = []
     for field in json.loads(cfg.get('Log', 'fieldname_order')):
+        print(field)
         if type(field) is list:
             to_add = cfg.get('Log', 'char_separator').join([ jsonattr[subField] for subField in field ])
         else:
@@ -113,23 +115,24 @@ def handler_attribute(jsonattr):
 
     #try to get coord
     if jsonattr['category'] == "Network activity":
-        getCoordAndPublish(jsonattr['value'], jsonattr['category'])
+        getCoordAndPublish(zmq_name, jsonattr['value'], jsonattr['category'])
 
-    publish_log('Attribute', to_push)
+    publish_log(zmq_name, 'Attribute', to_push)
 
 
-def process_log(event):
+def process_log(zmq_name, event):
     event = event.decode('utf8')
     topic, eventdata = event.split(' ', maxsplit=1)
     jsonevent = json.loads(eventdata)
-    dico_action[topic](jsonevent)
+    dico_action[topic](zmq_name, jsonevent)
 
 
-def main():
+def main(zmqName):
     while True:
         content = socket.recv()
         content.replace(b'\n', b'') # remove \n...
-        process_log(content)
+        zmq_name = zmqName
+        process_log(zmq_name, content)
 
 
 dico_action = {
@@ -144,6 +147,11 @@ dico_action = {
 
 
 if __name__ == "__main__":
-    main()
+
+    parser = argparse.ArgumentParser(description='A zmq subscriber. It subscribe to a ZNQ then redispatch it to the misp-dashboard')
+    parser.add_argument('-n', '--name', required=False, dest='zmqname', help='The ZMQ feed name', default="Misp Standard ZMQ")
+    args = parser.parse_args()
+
+    main(args.zmqname)
     reader.close()