diff --git a/README.md b/README.md index 8fd1f8d..b854bf4 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,16 @@ An experimental dashboard showing live data and statistics from the ZMQ of one o - RedisGlobal -> misp_web_url - RedisMap -> pathMaxMindDB +# Updating by pulling +- Re-launch ```./install_dependencies.sh``` to fetch new required dependencies +- Re-update your configuration file ```config.cfg``` + # Starting the System +- Be sure to have a running redis server + - e.g. ```redis-server -p 6250``` - Activate your virtualenv ```. ./DASHENV/bin/activate``` - Listen to the MISP feed by starting the zmq_subscriber ```./zmq_subscriber.py``` +- Start the dispatcher to process received messages ```./zmq_dispatcher.py``` - Start the Flask server ```./server.py``` - Access the interface at ```http://localhost:8001/``` diff --git a/config/config.cfg.default b/config/config.cfg.default index eabe996..6b3b46b 100644 --- a/config/config.cfg.default +++ b/config/config.cfg.default @@ -37,6 +37,10 @@ misp_web_url = http://localhost #zmq_url=tcp://192.168.56.50:50000 zmq_url=tcp://localhost:50000 +[RedisLIST] +db=3 +listName=bufferList + [RedisLog] db=0 channel=1 diff --git a/contributor_helper.py b/contributor_helper.py index 0f47d43..6924218 100644 --- a/contributor_helper.py +++ b/contributor_helper.py @@ -1,18 +1,30 @@ import util from util import getZrange import math, random +import time import os import configparser import json import datetime +import redis + +import util +import users_helper +KEYDAY = "CONTRIB_DAY" # To be used by other module class Contributor_helper: def __init__(self, serv_redis_db, cfg): self.serv_redis_db = serv_redis_db + self.serv_log = redis.StrictRedis( + host=cfg.get('RedisGlobal', 'host'), + port=cfg.getint('RedisGlobal', 'port'), + db=cfg.getint('RedisLog', 'db')) self.cfg = cfg self.cfg_org_rank = configparser.ConfigParser() self.cfg_org_rank.read(os.path.join(os.environ['DASH_CONFIG'], 'ranking.cfg')) self.CHANNEL_LASTAWARDS = cfg.get('RedisLog', 'channelLastAwards') + self.CHANNEL_LASTCONTRIB = cfg.get('RedisLog', 'channelLastContributor') + self.users_helper = users_helper.Users_helper(serv_redis_db, cfg) #honorBadge self.honorBadgeNum = len(self.cfg_org_rank.options('HonorBadge')) @@ -63,10 +75,10 @@ class Contributor_helper: self.DICO_PNTS_REWARD[categ] = self.default_pnts_per_contribution self.rankMultiplier = self.cfg_org_rank.getfloat('monthlyRanking' ,'rankMultiplier') - self.levelMax = self.cfg_org_rank.getfloat('monthlyRanking' ,'levelMax') + self.levelMax = self.cfg_org_rank.getint('monthlyRanking' ,'levelMax') # REDIS KEYS - self.keyDay = "CONTRIB_DAY" + self.keyDay = KEYDAY self.keyCateg = "CONTRIB_CATEG" self.keyLastContrib = "CONTRIB_LAST" self.keyAllOrg = "CONTRIB_ALL_ORG" @@ -86,7 +98,7 @@ class Contributor_helper: def publish_log(self, zmq_name, name, content, channel=""): to_send = { 'name': name, 'log': json.dumps(content), 'zmqName': zmq_name } - serv_log.publish(channel, json.dumps(to_send)) + self.serv_log.publish(channel, json.dumps(to_send)) ''' HANDLER ''' #pntMultiplier if one contribution rewards more than others. (e.g. shighting may gives more points than editing) @@ -100,36 +112,36 @@ class Contributor_helper: pnts_to_add = self.default_pnts_per_contribution # if there is a contribution, there is a login (even if ti comes from the API) - users_helper.add_user_login(nowSec, org) + self.users_helper.add_user_login(nowSec, org) # is a valid contribution if categ is not None: try: - pnts_to_add = self.DICO_PNTS_REWARD[noSpaceLower(categ)] + pnts_to_add = self.DICO_PNTS_REWARD[util.noSpaceLower(categ)] except KeyError: pnts_to_add = self.default_pnts_per_contribution pnts_to_add *= pntMultiplier util.push_to_redis_zset(self.serv_redis_db, self.keyDay, org, count=pnts_to_add) #CONTRIB_CATEG retain the contribution per category, not the point earned in this categ - util.push_to_redis_zset(self.serv_redis_db, self.keyCateg, org, count=1, endSubkey=':'+noSpaceLower(categ)) - self.publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'categ': categ, 'action': action, 'epoch': nowSec }, channel=CHANNEL_LASTCONTRIB) + util.push_to_redis_zset(self.serv_redis_db, self.keyCateg, org, count=1, endSubkey=':'+util.noSpaceLower(categ)) + self.publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'categ': categ, 'action': action, 'epoch': nowSec }, channel=self.CHANNEL_LASTCONTRIB) else: categ = "" - serv_redis_db.sadd(self.keyAllOrg, org) + self.serv_redis_db.sadd(self.keyAllOrg, org) keyname = "{}:{}".format(self.keyLastContrib, util.getDateStrFormat(now)) - serv_redis_db.zadd(keyname, nowSec, org) - serv_redis_db.expire(keyname, ONE_DAY*7) #expire after 7 day + self.serv_redis_db.zadd(keyname, nowSec, org) + self.serv_redis_db.expire(keyname, util.ONE_DAY*7) #expire after 7 day - awards_given = self.updateOrgContributionRank(org, pnts_to_add, action, contribType, eventTime=datetime.datetime.now(), isLabeled=isLabeled, categ=noSpaceLower(categ)) + awards_given = self.updateOrgContributionRank(org, pnts_to_add, action, contribType, eventTime=datetime.datetime.now(), isLabeled=isLabeled, categ=util.noSpaceLower(categ)) for award in awards_given: # update awards given keyname = "{}:{}".format(self.keyLastAward, util.getDateStrFormat(now)) - serv_redis_db.zadd(keyname, nowSec, json.dumps({'org': org, 'award': award, 'epoch': nowSec })) - serv_redis_db.expire(keyname, ONE_DAY*7) #expire after 7 day + self.serv_redis_db.zadd(keyname, nowSec, json.dumps({'org': org, 'award': award, 'epoch': nowSec })) + self.serv_redis_db.expire(keyname, util.ONE_DAY*7) #expire after 7 day # publish self.publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'award': award, 'epoch': nowSec }, channel=self.CHANNEL_LASTAWARDS) @@ -446,7 +458,7 @@ class Contributor_helper: dic['epoch'] = epoch return dic - def getTopContributorFromRedis(self, date): + def getTopContributorFromRedis(self, date, maxNum=100): orgDicoPnts = {} for curDate in util.getMonthSpan(date): topNum = 0 # all @@ -468,7 +480,7 @@ class Contributor_helper: data.append(dic) data.sort(key=lambda x: x['pnts'], reverse=True) - return data + return data[:maxNum] def getTop5OvertimeFromRedis(self): data = [] diff --git a/geo_helper.py b/geo_helper.py index ca1c83f..0a7f2da 100644 --- a/geo_helper.py +++ b/geo_helper.py @@ -109,14 +109,15 @@ class Geo_helper: def getCoordFromPhoneAndPublish(self, phoneNumber, categ): try: - print('function accessed') rep = phonenumbers.parse(phoneNumber, None) if not (phonenumbers.is_valid_number(rep) or phonenumbers.is_possible_number(rep)): print("Phone number not valid") + return country_name = geocoder.country_name_for_number(rep, "en") country_code = self.country_to_iso[country_name] if country_code is None: print("Non matching ISO_CODE") + return coord = self.country_code_to_coord[country_code.lower()] # countrycode is in upper case coord_dic = {'lat': coord['lat'], 'lon': coord['long']} @@ -139,7 +140,6 @@ class Geo_helper: "cityName": "", "regionCode": country_code, } - print(to_send) self.serv_coord.publish(self.CHANNELDISP, json.dumps(to_send)) except phonenumbers.NumberParseException: print("Can't resolve phone number country") diff --git a/server.py b/server.py index 9611bcc..6866f60 100755 --- a/server.py +++ b/server.py @@ -312,7 +312,7 @@ def eventStreamAwards(): yield 'data: {}\n\n'.format(json.dumps(to_return)) @app.route("/_getTopContributor") -def getTopContributor(suppliedDate=None): +def getTopContributor(suppliedDate=None, maxNum=100): if suppliedDate is None: try: date = datetime.datetime.fromtimestamp(float(request.args.get('date'))) @@ -321,7 +321,7 @@ def getTopContributor(suppliedDate=None): else: date = suppliedDate - data = contributor_helper.getTopContributorFromRedis(date) + data = contributor_helper.getTopContributorFromRedis(date, maxNum=maxNum) return jsonify(data) @app.route("/_getFameContributor") @@ -332,7 +332,7 @@ def getFameContributor(): today = datetime.datetime.now() # get previous month date = (datetime.datetime(today.year, today.month, 1) - datetime.timedelta(days=1)) - return getTopContributor(suppliedDate=date) + return getTopContributor(suppliedDate=date, maxNum=10) @app.route("/_getFameQualContributor") def getFameQualContributor(): @@ -342,7 +342,7 @@ def getFameQualContributor(): today = datetime.datetime.now() # get previous month date = (datetime.datetime(today.year, today.month, 1) - datetime.timedelta(days=1)) - return getTopContributor(suppliedDate=date) + return getTopContributor(suppliedDate=date, maxNum=10) @app.route("/_getTop5Overtime") def getTop5Overtime(): diff --git a/start.sh b/start_all.sh similarity index 78% rename from start.sh rename to start_all.sh index 7371498..1296c21 100755 --- a/start.sh +++ b/start_all.sh @@ -15,10 +15,13 @@ screenName="Misp-Dashboard" screen -dmS "$screenName" sleep 0.1 echo -e $GREEN"\t* Launching Redis servers"$DEFAULT -screen -S "$screenName" -X screen -t "redis-server" bash -c $redis_dir'redis-server '$conf_dir'6250.conf ; read x' +screen -S "$screenName" -X screen -t "redis-server" bash -c $redis_dir'redis-server '$conf_dir'6250.conf; read x' echo -e $GREEN"\t* Launching zmq subscriber"$DEFAULT screen -S "$screenName" -X screen -t "zmq-subscriber" bash -c './zmq_subscriber.py; read x' +echo -e $GREEN"\t* Launching zmq dispatcher"$DEFAULT +screen -S "$screenName" -X screen -t "zmq-dispatcher" bash -c './zmq_dispatcher.py; read x' + echo -e $GREEN"\t* Launching flask server"$DEFAULT screen -S "$screenName" -X screen -t "flask" bash -c './server.py; read x' diff --git a/static/css/ranking.css b/static/css/ranking.css index df7d62d..f0116e6 100644 --- a/static/css/ranking.css +++ b/static/css/ranking.css @@ -17,10 +17,10 @@ } .circleBadgeSmall { - width: 73px; - height: 73px; + width: 57px; + height: 57px; text-align: center; - border-radius: 38px; + border-radius: 28px; background-color: #e1e1e1; border: 1px solid #caccce; vertical-align: middle; diff --git a/static/js/contrib.js b/static/js/contrib.js index 2809911..e0d3754 100644 --- a/static/js/contrib.js +++ b/static/js/contrib.js @@ -79,8 +79,9 @@ optionDatatable_last.columnDefs = [ { className: "centerCellPicOrgLogo verticalAlign", "targets": [ 5 ] }, { className: "verticalAlign", "targets": [ 6 ] } ] -var optionDatatable_fame = jQuery.extend({}, optionDatatable_light) -optionDatatable_fame.scrollY = '45vh'; +var optionDatatable_fameQuant = jQuery.extend({}, optionDatatable_light) +var optionDatatable_fameQual = jQuery.extend({}, optionDatatable_light) +optionDatatable_fameQual.scrollY = '39vh'; var optionDatatable_Categ = { responsive: true, @@ -590,8 +591,8 @@ $(document).ready(function() { }); datatableTop = $('#topContribTable').DataTable(optionDatatable_top); - datatableFameQuant = $('#fameTableQuantity').DataTable(optionDatatable_fame); - datatableFameQual = $('#fameTableQuality').DataTable(optionDatatable_fame); + datatableFameQuant = $('#fameTableQuantity').DataTable(optionDatatable_fameQuant); + datatableFameQual = $('#fameTableQuality').DataTable(optionDatatable_fameQual); datatableCateg = $('#categTable').DataTable(optionDatatable_Categ); datatableLast = $('#lastTable').DataTable(optionDatatable_last); datatableAwards = $('#awardTable').DataTable(optionDatatable_awards); diff --git a/templates/contrib.html b/templates/contrib.html index 4faee41..f54a5c2 100644 --- a/templates/contrib.html +++ b/templates/contrib.html @@ -172,7 +172,7 @@