Started support of MISP ZMQ

config.cfg

@@ -3,7 +3,7 @@
 graph_log_refresh_rate = 1
 
 [Log]
-fieldname_order=["time", "level", "name", "message"]
+fieldname_order=["time", "id", "category", "type", "value"]
 
 [RedisLog]
 host=localhost

retreive_map_pic.py

@@ -70,7 +70,7 @@ def download_maps(coord):
     urls = create_box_around_coord(lon, lat)
     map_name = download_and_merge(urls, coord)
     print(map_name)
-    serv.publish(channel_disp, map_name)
+    serv.publish(channel_disp, json.dumps({ "path": map_name, "coord": coord }))
 
 if __name__ == '__main__':
     main()

server.py

@@ -100,13 +100,13 @@ def getLogHead():
 def event_stream_log():
     for msg in subscriber_log.listen():
         content = msg['data']
-        print('sending', content)
         yield EventMessage(content).to_json()
 
 def event_stream_maps():
     for msg in subscriber_map.listen():
         content = msg['data'].decode('utf8')
-        yield 'data: {}\n\n'.format(json.dumps({ 'path': content }))
+        print('sending map', content)
+        yield 'data: {}\n\n'.format(content)
 
 if __name__ == '__main__':
     app.run(host='localhost', port=8000, threaded=True)

static/js/index.js

@@ -166,6 +166,7 @@ $(document).ready(function () {
     var source_map = new EventSource(urlForMaps);
     source_map.onmessage = function(event) {
         var json = jQuery.parseJSON( event.data );
+        popupCoord(json.coord);
         var img2 = linkForDefaultMap.replace(/\/[^\/]+$/, "/"+json.path);
         $("#img"+img_to_change).fadeOut(400, function(){ $(this).attr('src', img2); }).fadeIn(400);
         $("#textMap"+img_to_change).fadeOut(400, function(){ $(this).text(json.path); }).fadeIn(400);
@@ -219,20 +220,27 @@ function updateLogTable(feedName, log) {
     sources.addIfNotPresent(feedName);
     sources.incCountOnSource(feedName);
     sources.incCountOnSource('global');
-    createRow(tableBody, log);
 
-    // Remove old row
+    // only add row for attribute
-    var logSel = document.getElementById("log_select");
+    if (feedName == "Attribute" ) {
-    //get height of pannel, find max num of item
+        createRow(tableBody, log);
-    var maxNumLogItem = document.getElementById('divLogTable').clientHeight/37;
+
-    maxNumLogItem -= 2; //take heading/padding/... into account
+        // Remove old row
-    if (maxNumLogItem - parseInt(maxNumLogItem) < 0.5) { //beautifier
+        var logSel = document.getElementById("log_select");
-        maxNumLogItem -= 1;
+        //get height of pannel, find max num of item
-    }
+        var maxNumLogItem = document.getElementById('divLogTable').clientHeight/37;
-    if (tableBody.rows.length > maxNumLogItem) {
+        maxNumLogItem -= 2; //take heading/padding/... into account
-        while (tableBody.rows.length >= maxNumLogItem){
+        if (maxNumLogItem - parseInt(maxNumLogItem) < 0.5) { //beautifier
-            tableBody.deleteRow(0);
+            maxNumLogItem -= 1;
         }
+        if (tableBody.rows.length > maxNumLogItem) {
+            while (tableBody.rows.length >= maxNumLogItem){
+                tableBody.deleteRow(0);
+            }
+        }
+    } else {
+        // do nothing
+        return;
     }
 
 }
@@ -252,10 +260,7 @@ function slideAndMax(orig, newData) {
 
 function createRow(tableBody, log) {
     var tr = document.createElement('TR');
-    var action = document.createElement('TD');
+    //var action = document.createElement('TD');
-    var x = log[1];
-    var y = log[2];
-    popupCoord([x,y]);
 
     for (var key in log) {
         if (log.hasOwnProperty(key)) {
@@ -277,8 +282,8 @@ function createRow(tableBody, log) {
     }
 
     // action
-    action.appendChild(document.createTextNode("ACTION"));
+    //action.appendChild(document.createTextNode("ACTION"));
-    tr.appendChild(action);
+    //tr.appendChild(action);
 
     tableBody.appendChild(tr);
 
@@ -294,9 +299,9 @@ function createHead(callback) {
             th.appendChild(document.createTextNode(head));
             tr.appendChild(th);
         }
-        var action = document.createElement('TH');
+        //var action = document.createElement('TH');
-        action.appendChild(document.createTextNode("Actions"));
+        //action.appendChild(document.createTextNode("Actions"));
-        tr.appendChild(action);
+        //tr.appendChild(action);
         document.getElementById('table_log_head').appendChild(tr);
         callback();
     });
@@ -320,7 +325,8 @@ function marker_animation(x, y, curNumMarker) {
 }
 
 function popupCoord(coord) {
-    var value = coord[0]+coord[1];
+    var coord = [coord['lat'], coord['lon']];
+    var value = Math.random()*180;
     pnts = mapObj.latLngToPoint(coord[0], coord[1])
     if (pnts != false) { //sometimes latLngToPoint return false
         mapObj.addMarker(curNumMarker, coord, [value]);
@@ -345,7 +351,7 @@ $(function(){
             attribute: 'fill',
             scale: ['#1A0DAB', '#e50000', '#62ff41'],
             values: [],
-            min: -180,
+            min: 0,
             max: 180
           }],
         },

templates/index.html

@@ -97,6 +97,33 @@ small {
                             </div>
                             <!-- /.panel-body -->
 
+                        </div>
+                        <!-- /.panel -->
+                        <div class="panel panel-default" style="margin-top: 15px; height: 29vh;">
+                            <div class="panel-heading">
+                                <i class="fa fa-bar-chart-o fa-fw"></i> Log feed
+                            </div>
+                            <div id="panelbody" class="panel-body" style="width:100%; height: calc(100% - 30px);">
+                                <div id="feedDiv3" style="width:100%; height: calc(100% - 30px); position: relative;"></div>
+                            </div>
+                        </div>
+
+
+                    </div>
+                    <!-- /.col-lg-6 -->
+                    <!-- /.col-lg-6 -->
+                    <!-- vh sum = 87 -->
+                    <div class="col-lg-7">
+                        
+                        <div class="panel panel-default" style="margin-top: 15px; height: 59vh;">
+                            <!--<div class="panel-heading">
+                                <i class="fa fa-bar-chart-o fa-fw"></i> Feed
+                            </div>-->
+                            <div id="panelbody" class="panel-body" style="height: 100%;">
+                                <div id="feedDiv2" style="width:100%; height: calc(100% - 30px); position: relative;"></div>
+                            </div>
+                            <!-- /.panel-body -->
+
                         </div>
                         <!-- /.panel -->
 
@@ -122,32 +149,6 @@ small {
                                 </div>
                             </div>
                         </div>
-                    </div>
-                    <!-- /.col-lg-6 -->
-                    <!-- /.col-lg-6 -->
-                    <!-- vh sum = 87 -->
-                    <div class="col-lg-7">
-                        
-                        <div class="panel panel-default" style="margin-top: 15px; height: 59vh;">
-                            <!--<div class="panel-heading">
-                                <i class="fa fa-bar-chart-o fa-fw"></i> Feed
-                            </div>-->
-                            <div id="panelbody" class="panel-body" style="height: 100%;">
-                                <div id="feedDiv2" style="width:100%; height: calc(100% - 30px); position: relative;"></div>
-                            </div>
-                            <!-- /.panel-body -->
-
-                        </div>
-                        <!-- /.panel -->
-
-                        <div class="panel panel-default" style="margin-top: 15px; height: 29vh;">
-                            <div class="panel-heading">
-                                <i class="fa fa-bar-chart-o fa-fw"></i> Log feed
-                            </div>
-                            <div id="panelbody" class="panel-body" style="width:100%; height: calc(100% - 30px);">
-                                <div id="feedDiv3" style="width:100%; height: calc(100% - 30px); position: relative;"></div>
-                            </div>
-                        </div>
 
                     </div>
                     <!-- /.col-lg-6 -->

zmq_subscriber.py

@@ -8,18 +8,20 @@ import configparser
 import os
 import sys
 import json
+import geoip2.database
 
 configfile = os.path.join(os.environ['VIRTUAL_ENV'], '../config.cfg')
 cfg = configparser.ConfigParser()
 cfg.read(configfile)
 
 zmq_url = cfg.get('RedisLog', 'zmq_url')
-zmq_url = "tcp://crf.circl.lu:5556"
+zmq_url = "tcp://192.168.56.50:50000"
+zmq_url = "tcp://localhost:9990"
 channel = cfg.get('RedisLog', 'channel')
 context = zmq.Context()
 socket = context.socket(zmq.SUB)
 socket.connect(zmq_url)
-socket.setsockopt_string(zmq.SUBSCRIBE, channel)
+socket.setsockopt_string(zmq.SUBSCRIBE, '')
 
 redis_server = redis.StrictRedis(
         host=cfg.get('RedisLog', 'host'),
@@ -29,42 +31,138 @@ serv_coord = redis.StrictRedis(
         host='localhost',
         port=6250,
         db=1) 
+path_to_db = "/home/sami/Downloads/GeoLite2-City_20171003/GeoLite2-City.mmdb"
+reader = geoip2.database.Reader(path_to_db)
 
 channel_proc = "CoordToProcess"
 channel_disp = "PicToDisplay"
 
+
+def publish_coord(coord):
+    pass
+
+def get_ip(data):
+    pass
+
+def ip_to_coord(ip):
+    resp = reader.city(ip)
+    lat = resp.location.latitude
+    lon = resp.location.longitude
+    return {'lat': lat, 'lon': lon}
+
+def default_log(jsonevent):
+    print('sending', 'log')
+    return
+    #redis_server.publish(channel, json.dumps(jsonevent))
+
+def default_keepalive(jsonevent):
+    print('sending', 'keepalive')
+    to_push = [ jsonevent['uptime'] ]
+    to_send = { 'name': 'Keepalive', 'log': json.dumps(to_push) }
+    redis_server.publish(channel, json.dumps(to_send))
+
+def default_event(jsonevent):
+    print('sending', 'event')
+    jsonevent = jsonevent['Event']
+    to_push = [
+            jsonevent['threat_level_id'],
+            jsonevent['id'],
+            jsonevent['info'],
+            ]
+    to_send = { 'name': 'Event', 'log': json.dumps(to_push) }
+    redis_server.publish(channel, json.dumps(to_send))
+
+def default_attribute(jsonevent):
+    print('sending', 'attribute')
+    jsonevent = jsonevent['Attribute']
+    to_push = [
+            jsonevent['id'],
+            jsonevent['category'],
+            jsonevent['type'],
+            jsonevent['value'],
+            ]
+
+    #try to get coord
+    if jsonevent['category'] == "Network activity":
+        try:
+            coord = ip_to_coord(jsonevent['value'])
+            to_send = {'lat': float(coord['lat']), 'lon': float(coord['lon'])}
+            serv_coord.publish(channel_proc, json.dumps(to_send))
+            print('coord sent')
+        except ValueError:
+            print("can't resolve ip")
+
+    to_send = { 'name': 'Attribute', 'log': json.dumps(to_push) }
+    redis_server.publish(channel, json.dumps(to_send))
+
+
+def process_log(event):
+    event = event.decode('utf8')
+    topic, eventdata = event.split(' ', maxsplit=1)
+    jsonevent = json.loads(eventdata)
+    dico_action[topic](jsonevent)
+
+
+def main():
+    while True:
+        content = socket.recv()
+        content.replace(b'\n', b'') # remove \n...
+        process_log(content)
+
+def log_feed():
+    with open('misp-zmq.2', 'ba') as f:
+    
+        while True:
+            time.sleep(1.0)
+            content = socket.recv()
+            content.replace(b'\n', b'') # remove \n...
+            f.write(content)
+            f.write(b'\n')
+            print(content)
+            #redis_server.publish(channel, content)
+        
+            #if random.randint(1,10)<5:
+            #    time.sleep(0.5)
+            #    redis_server.publish(channel, content)
+        
+            #if random.randint(1,10)<5:
+            #    time.sleep(0.5)
+            #    redis_server.publish(channel, content)
+
+dico_action = {
+        "misp_json":                default_event,
+        "misp_json_self":           default_keepalive,
+        "misp_json_attribute":      default_attribute,
+        "misp_json_sighting":       default_log,
+        "misp_json_organisation":   default_log,
+        "misp_json_user":           default_log,
+        "misp_json_conversation":   default_log
+        }
+
+
+if __name__ == "__main__":
+    main()
+    reader.close()
+
+
+
 # server side
 pubsub = redis_server.pubsub(ignore_subscribe_messages=True)
 
-while True:
+#while True:
-    rdm = random.randint(1,10)
+#    rdm = random.randint(1,10)
-    time.sleep(float(rdm))
+#    time.sleep(float(rdm))
-    #lux
+#    #lux
-    lon = random.uniform(5.7373, 6.4823)
+#    lon = random.uniform(5.7373, 6.4823)
-    lat = random.uniform(49.4061,49.7449)
+#    lat = random.uniform(49.4061,49.7449)
-    #central eur
+#    #central eur
-    lon = random.uniform(3.936, 9.890)
+#    lon = random.uniform(3.936, 9.890)
-    lat = random.uniform(47.957, 50.999)
+#    lat = random.uniform(47.957, 50.999)
-    content = ["rdm "+str(rdm)]
+#    content = ["rdm "+str(rdm)]
-    content = [lat,lon]
+#    content = [lat,lon]
-    jsonContent = json.dumps(content)
+#    jsonContent = json.dumps(content)
-    to_send = { 'name': 'feeder'+str(rdm), 'log': jsonContent }
+#    to_send = { 'name': 'feeder'+str(rdm), 'log': jsonContent }
-    redis_server.publish(channel, json.dumps(to_send))
+#    redis_server.publish(channel, json.dumps(to_send))
-    serv_coord.publish(channel_proc, json.dumps({'lat': float(lat), 'lon': float(lon)}))
+#    serv_coord.publish(channel_proc, json.dumps({'lat': float(lat), 'lon': float(lon)}))
 
 
-
-while True:
-    #FIXME check if sock.recv is blocking
-    time.sleep(0.1)
-    content = socket.recv()
-    print('sending', (content))
-    redis_server.publish(channel, content)
-
-    if random.randint(1,10)<5:
-        time.sleep(0.5)
-        redis_server.publish(channel, content)
-
-    if random.randint(1,10)<5:
-        time.sleep(0.5)
-        redis_server.publish(channel, content)