diff --git a/config.cfg b/config.cfg
index d6467a9..1375df8 100644
--- a/config.cfg
+++ b/config.cfg
@@ -3,7 +3,7 @@
graph_log_refresh_rate = 1
[Log]
-fieldname_order=["time", "level", "name", "message"]
+fieldname_order=["time", "id", "category", "type", "value"]
[RedisLog]
host=localhost
diff --git a/retreive_map_pic.py b/retreive_map_pic.py
index 96d30b0..55929a1 100755
--- a/retreive_map_pic.py
+++ b/retreive_map_pic.py
@@ -70,7 +70,7 @@ def download_maps(coord):
urls = create_box_around_coord(lon, lat)
map_name = download_and_merge(urls, coord)
print(map_name)
- serv.publish(channel_disp, map_name)
+ serv.publish(channel_disp, json.dumps({ "path": map_name, "coord": coord }))
if __name__ == '__main__':
main()
diff --git a/server.py b/server.py
index fc82f05..8a7eaaf 100755
--- a/server.py
+++ b/server.py
@@ -100,13 +100,13 @@ def getLogHead():
def event_stream_log():
for msg in subscriber_log.listen():
content = msg['data']
- print('sending', content)
yield EventMessage(content).to_json()
def event_stream_maps():
for msg in subscriber_map.listen():
content = msg['data'].decode('utf8')
- yield 'data: {}\n\n'.format(json.dumps({ 'path': content }))
+ print('sending map', content)
+ yield 'data: {}\n\n'.format(content)
if __name__ == '__main__':
app.run(host='localhost', port=8000, threaded=True)
diff --git a/static/js/index.js b/static/js/index.js
index 1226f21..e9d317a 100644
--- a/static/js/index.js
+++ b/static/js/index.js
@@ -166,6 +166,7 @@ $(document).ready(function () {
var source_map = new EventSource(urlForMaps);
source_map.onmessage = function(event) {
var json = jQuery.parseJSON( event.data );
+ popupCoord(json.coord);
var img2 = linkForDefaultMap.replace(/\/[^\/]+$/, "/"+json.path);
$("#img"+img_to_change).fadeOut(400, function(){ $(this).attr('src', img2); }).fadeIn(400);
$("#textMap"+img_to_change).fadeOut(400, function(){ $(this).text(json.path); }).fadeIn(400);
@@ -219,20 +220,27 @@ function updateLogTable(feedName, log) {
sources.addIfNotPresent(feedName);
sources.incCountOnSource(feedName);
sources.incCountOnSource('global');
- createRow(tableBody, log);
- // Remove old row
- var logSel = document.getElementById("log_select");
- //get height of pannel, find max num of item
- var maxNumLogItem = document.getElementById('divLogTable').clientHeight/37;
- maxNumLogItem -= 2; //take heading/padding/... into account
- if (maxNumLogItem - parseInt(maxNumLogItem) < 0.5) { //beautifier
- maxNumLogItem -= 1;
- }
- if (tableBody.rows.length > maxNumLogItem) {
- while (tableBody.rows.length >= maxNumLogItem){
- tableBody.deleteRow(0);
+ // only add row for attribute
+ if (feedName == "Attribute" ) {
+ createRow(tableBody, log);
+
+ // Remove old row
+ var logSel = document.getElementById("log_select");
+ //get height of pannel, find max num of item
+ var maxNumLogItem = document.getElementById('divLogTable').clientHeight/37;
+ maxNumLogItem -= 2; //take heading/padding/... into account
+ if (maxNumLogItem - parseInt(maxNumLogItem) < 0.5) { //beautifier
+ maxNumLogItem -= 1;
}
+ if (tableBody.rows.length > maxNumLogItem) {
+ while (tableBody.rows.length >= maxNumLogItem){
+ tableBody.deleteRow(0);
+ }
+ }
+ } else {
+ // do nothing
+ return;
}
}
@@ -252,10 +260,7 @@ function slideAndMax(orig, newData) {
function createRow(tableBody, log) {
var tr = document.createElement('TR');
- var action = document.createElement('TD');
- var x = log[1];
- var y = log[2];
- popupCoord([x,y]);
+ //var action = document.createElement('TD');
for (var key in log) {
if (log.hasOwnProperty(key)) {
@@ -277,8 +282,8 @@ function createRow(tableBody, log) {
}
// action
- action.appendChild(document.createTextNode("ACTION"));
- tr.appendChild(action);
+ //action.appendChild(document.createTextNode("ACTION"));
+ //tr.appendChild(action);
tableBody.appendChild(tr);
@@ -294,9 +299,9 @@ function createHead(callback) {
th.appendChild(document.createTextNode(head));
tr.appendChild(th);
}
- var action = document.createElement('TH');
- action.appendChild(document.createTextNode("Actions"));
- tr.appendChild(action);
+ //var action = document.createElement('TH');
+ //action.appendChild(document.createTextNode("Actions"));
+ //tr.appendChild(action);
document.getElementById('table_log_head').appendChild(tr);
callback();
});
@@ -320,7 +325,8 @@ function marker_animation(x, y, curNumMarker) {
}
function popupCoord(coord) {
- var value = coord[0]+coord[1];
+ var coord = [coord['lat'], coord['lon']];
+ var value = Math.random()*180;
pnts = mapObj.latLngToPoint(coord[0], coord[1])
if (pnts != false) { //sometimes latLngToPoint return false
mapObj.addMarker(curNumMarker, coord, [value]);
@@ -345,7 +351,7 @@ $(function(){
attribute: 'fill',
scale: ['#1A0DAB', '#e50000', '#62ff41'],
values: [],
- min: -180,
+ min: 0,
max: 180
}],
},
diff --git a/templates/index.html b/templates/index.html
index dbbfc59..7ec0265 100644
--- a/templates/index.html
+++ b/templates/index.html
@@ -97,6 +97,33 @@ small {
+
+
+
+
+
+
+
+
+
+
+
+
@@ -122,32 +149,6 @@ small {
-
-
-
-
-
diff --git a/zmq_subscriber.py b/zmq_subscriber.py
index d9120a8..4b80161 100755
--- a/zmq_subscriber.py
+++ b/zmq_subscriber.py
@@ -8,18 +8,20 @@ import configparser
import os
import sys
import json
+import geoip2.database
configfile = os.path.join(os.environ['VIRTUAL_ENV'], '../config.cfg')
cfg = configparser.ConfigParser()
cfg.read(configfile)
zmq_url = cfg.get('RedisLog', 'zmq_url')
-zmq_url = "tcp://crf.circl.lu:5556"
+zmq_url = "tcp://192.168.56.50:50000"
+zmq_url = "tcp://localhost:9990"
channel = cfg.get('RedisLog', 'channel')
context = zmq.Context()
socket = context.socket(zmq.SUB)
socket.connect(zmq_url)
-socket.setsockopt_string(zmq.SUBSCRIBE, channel)
+socket.setsockopt_string(zmq.SUBSCRIBE, '')
redis_server = redis.StrictRedis(
host=cfg.get('RedisLog', 'host'),
@@ -29,42 +31,138 @@ serv_coord = redis.StrictRedis(
host='localhost',
port=6250,
db=1)
+path_to_db = "/home/sami/Downloads/GeoLite2-City_20171003/GeoLite2-City.mmdb"
+reader = geoip2.database.Reader(path_to_db)
channel_proc = "CoordToProcess"
channel_disp = "PicToDisplay"
+
+def publish_coord(coord):
+ pass
+
+def get_ip(data):
+ pass
+
+def ip_to_coord(ip):
+ resp = reader.city(ip)
+ lat = resp.location.latitude
+ lon = resp.location.longitude
+ return {'lat': lat, 'lon': lon}
+
+def default_log(jsonevent):
+ print('sending', 'log')
+ return
+ #redis_server.publish(channel, json.dumps(jsonevent))
+
+def default_keepalive(jsonevent):
+ print('sending', 'keepalive')
+ to_push = [ jsonevent['uptime'] ]
+ to_send = { 'name': 'Keepalive', 'log': json.dumps(to_push) }
+ redis_server.publish(channel, json.dumps(to_send))
+
+def default_event(jsonevent):
+ print('sending', 'event')
+ jsonevent = jsonevent['Event']
+ to_push = [
+ jsonevent['threat_level_id'],
+ jsonevent['id'],
+ jsonevent['info'],
+ ]
+ to_send = { 'name': 'Event', 'log': json.dumps(to_push) }
+ redis_server.publish(channel, json.dumps(to_send))
+
+def default_attribute(jsonevent):
+ print('sending', 'attribute')
+ jsonevent = jsonevent['Attribute']
+ to_push = [
+ jsonevent['id'],
+ jsonevent['category'],
+ jsonevent['type'],
+ jsonevent['value'],
+ ]
+
+ #try to get coord
+ if jsonevent['category'] == "Network activity":
+ try:
+ coord = ip_to_coord(jsonevent['value'])
+ to_send = {'lat': float(coord['lat']), 'lon': float(coord['lon'])}
+ serv_coord.publish(channel_proc, json.dumps(to_send))
+ print('coord sent')
+ except ValueError:
+ print("can't resolve ip")
+
+ to_send = { 'name': 'Attribute', 'log': json.dumps(to_push) }
+ redis_server.publish(channel, json.dumps(to_send))
+
+
+def process_log(event):
+ event = event.decode('utf8')
+ topic, eventdata = event.split(' ', maxsplit=1)
+ jsonevent = json.loads(eventdata)
+ dico_action[topic](jsonevent)
+
+
+def main():
+ while True:
+ content = socket.recv()
+ content.replace(b'\n', b'') # remove \n...
+ process_log(content)
+
+def log_feed():
+ with open('misp-zmq.2', 'ba') as f:
+
+ while True:
+ time.sleep(1.0)
+ content = socket.recv()
+ content.replace(b'\n', b'') # remove \n...
+ f.write(content)
+ f.write(b'\n')
+ print(content)
+ #redis_server.publish(channel, content)
+
+ #if random.randint(1,10)<5:
+ # time.sleep(0.5)
+ # redis_server.publish(channel, content)
+
+ #if random.randint(1,10)<5:
+ # time.sleep(0.5)
+ # redis_server.publish(channel, content)
+
+dico_action = {
+ "misp_json": default_event,
+ "misp_json_self": default_keepalive,
+ "misp_json_attribute": default_attribute,
+ "misp_json_sighting": default_log,
+ "misp_json_organisation": default_log,
+ "misp_json_user": default_log,
+ "misp_json_conversation": default_log
+ }
+
+
+if __name__ == "__main__":
+ main()
+ reader.close()
+
+
+
# server side
pubsub = redis_server.pubsub(ignore_subscribe_messages=True)
-while True:
- rdm = random.randint(1,10)
- time.sleep(float(rdm))
- #lux
- lon = random.uniform(5.7373, 6.4823)
- lat = random.uniform(49.4061,49.7449)
- #central eur
- lon = random.uniform(3.936, 9.890)
- lat = random.uniform(47.957, 50.999)
- content = ["rdm "+str(rdm)]
- content = [lat,lon]
- jsonContent = json.dumps(content)
- to_send = { 'name': 'feeder'+str(rdm), 'log': jsonContent }
- redis_server.publish(channel, json.dumps(to_send))
- serv_coord.publish(channel_proc, json.dumps({'lat': float(lat), 'lon': float(lon)}))
+#while True:
+# rdm = random.randint(1,10)
+# time.sleep(float(rdm))
+# #lux
+# lon = random.uniform(5.7373, 6.4823)
+# lat = random.uniform(49.4061,49.7449)
+# #central eur
+# lon = random.uniform(3.936, 9.890)
+# lat = random.uniform(47.957, 50.999)
+# content = ["rdm "+str(rdm)]
+# content = [lat,lon]
+# jsonContent = json.dumps(content)
+# to_send = { 'name': 'feeder'+str(rdm), 'log': jsonContent }
+# redis_server.publish(channel, json.dumps(to_send))
+# serv_coord.publish(channel_proc, json.dumps({'lat': float(lat), 'lon': float(lon)}))
-
-while True:
- #FIXME check if sock.recv is blocking
- time.sleep(0.1)
- content = socket.recv()
- print('sending', (content))
- redis_server.publish(channel, content)
-
- if random.randint(1,10)<5:
- time.sleep(0.5)
- redis_server.publish(channel, content)
-
- if random.randint(1,10)<5:
- time.sleep(0.5)
- redis_server.publish(channel, content)