Merge pull request #121 from mokaddem/fewFixes2

Various fixes and improvements

diagnostic.py

@@ -18,7 +18,7 @@ try:
     from halo import Halo
 except ModuleNotFoundError as e:
     print('Dependency not met. Either not in a virtualenv or dependency not installed.')
-    print(f'- Error: {e}')
+    print('- Error: {}'.format(e))
     sys.exit(1)
 
 '''
@@ -79,12 +79,12 @@ def add_spinner(_func=None, name='dots'):
                 else:
                     status = False
                     flag_skip = True
-                    spinner.fail(f'{human_func_name} -  Function return unexpected result: {str(result)}')
+                    spinner.fail('{} -  Function return unexpected result: {}'.format(human_func_name, str(result)))
 
                 if not flag_skip:
                     text = human_func_result
                     if output is not None and len(output) > 0:
-                        text += f': {output}'
+                        text += ': {}'.format(output)
 
                     if isinstance(status, bool) and status:
                         spinner.succeed(text)
@@ -111,8 +111,8 @@ def check_virtual_environment_and_packages(spinner):
         return (False, 'This diagnostic tool should be started inside a virtual environment.')
     else:
         if redis.__version__.startswith('2'):
-            return (False, f'''Redis python client have version {redis.__version__}. Version 3.x required.
+            return (False, '''Redis python client have version {}. Version 3.x required.
-\t➥ [inside virtualenv] pip3 install -U redis''')
+\t➥ [inside virtualenv] pip3 install -U redis'''.format(redis.__version__))
         else:
             return (True, '')
 
@@ -139,7 +139,7 @@ def check_configuration(spinner):
         return_text = '''Configuration incomplete.
 \tUpdate your configuration file `config.cfg`.\n\t➥ Faulty fields:\n'''
         for field_name in faulties:
-            return_text += f'\t\t- {field_name}\n'
+            return_text += '\t\t- {}\n'.format(field_name)
         return (False, return_text)
 
 
@@ -192,7 +192,7 @@ def check_zmq(spinner):
                         flag_skip = True
                         break
             else:
-                spinner.text = f'checking zmq of {misp_instance.get("name")} - elapsed time: {int(time.time() - start_time)}s'
+                spinner.text = 'checking zmq of {} - elapsed time: {}s'.format(misp_instance.get("name"), int(time.time() - start_time))
         if not flag_skip:
             instances_status[misp_instance.get('name')] = False
 
@@ -202,7 +202,7 @@ def check_zmq(spinner):
     elif any(results):
         return_text = 'Connection to ZMQ stream(s) failed.\n'
         for name, status in instances_status.items():
-            return_text += f'\t➥ {name}: {"success" if status else "failed"}\n'
+            return_text += '\t➥ {}: {}\n'.format(name, "success" if status else "failed")
         return (True, return_text)
     else:
         return (False, '''Can\'t connect to the ZMQ stream(s).
@@ -257,14 +257,14 @@ def check_subscriber_status(spinner):
                 target = split[4]
             except IndexError:
                 pass
-            if action == '"LPUSH"' and target == f'\"{configuration_file.get("RedisLIST", "listName")}\"':
+            if action == '"LPUSH"' and target == '\"{}\"'.format(configuration_file.get("RedisLIST", "listName")):
                 signal.alarm(0)
                 break
             else:
-                spinner.text = f'Checking subscriber status - elapsed time: {int(time.time() - start_time)}s'
+                spinner.text = 'Checking subscriber status - elapsed time: {}s'.format(int(time.time() - start_time))
     except diagnostic_util.TimeoutException:
-        return_text = f'''zmq_subscriber seems not to be working.
+        return_text = '''zmq_subscriber seems not to be working.
-\t➥ Consider restarting it: {pgrep_subscriber_output}'''
+\t➥ Consider restarting it: {}'''.format(pgrep_subscriber_output)
         return (False, return_text)
     return (True, 'subscriber is running and populating the buffer')
 
@@ -278,7 +278,7 @@ def check_buffer_queue(spinner):
     warning_threshold = 100
     elements_in_list = redis_server.llen(configuration_file.get('RedisLIST', 'listName'))
     return_status = 'warning' if elements_in_list > warning_threshold else ('info' if elements_in_list > 0 else True)
-    return_text = f'Currently {elements_in_list} items in the buffer'
+    return_text = 'Currently {} items in the buffer'.format(elements_in_list)
     return (return_status, return_text)
 
 
@@ -308,8 +308,8 @@ def check_buffer_change_rate(spinner):
 
         if next_refresh < time_slept:
             next_refresh = time_slept + refresh_frequency
-            change_rate_text = f'↑ {change_increase}/sec\t↓ {change_decrease}/sec'
+            change_rate_text = '↑ {}/sec\t↓ {}/sec'.format(change_increase, change_decrease)
-            spinner.text = f'Buffer: {elements_in_list}\t{change_rate_text}'
+            spinner.text = 'Buffer: {}\t{}'.format(elements_in_list, change_rate_text)
 
             if consecutive_no_rate_change == 3:
                 time_slept = sleep_max
@@ -322,7 +322,7 @@ def check_buffer_change_rate(spinner):
 
         if time_slept >= sleep_max:
             return_flag = elements_in_list == 0 or (elements_in_list < elements_in_inlist_init or elements_in_list < 2)
-            return_text = f'Buffer is consumed {"faster" if return_flag else "slower" } than being populated'
+            return_text = 'Buffer is consumed {} than being populated'.format("faster" if return_flag else "slower")
             break
 
         time.sleep(sleep_duration)
@@ -354,18 +354,18 @@ def check_dispatcher_status(spinner):
         if reply is None:
             if time_slept >= sleep_max:
                 return_flag = False
-                return_text = f'zmq_dispatcher did not respond in the given time ({int(sleep_max)}s)'
+                return_text = 'zmq_dispatcher did not respond in the given time ({}s)'.format(int(sleep_max))
                 if len(pgrep_dispatcher_output) > 0:
-                    return_text += f'\n\t➥ Consider restarting it: {pgrep_dispatcher_output}'
+                    return_text += '\n\t➥ Consider restarting it: {}'.format(pgrep_dispatcher_output)
                 else:
                     return_text += '\n\t➥ Consider starting it'
                 break
             time.sleep(sleep_duration)
-            spinner.text = f'Dispatcher status: No response yet'
+            spinner.text = 'Dispatcher status: No response yet'
             time_slept += sleep_duration
         else:
             return_flag = True
-            return_text = f'Took {float(reply):.2f}s to complete'
+            return_text = 'Took {:.2f}s to complete'.format(float(reply))
             break
 
     return (return_flag, return_text)
@@ -373,15 +373,15 @@ def check_dispatcher_status(spinner):
 
 @add_spinner
 def check_server_listening(spinner):
-    url = f'{HOST}:{PORT}/_get_log_head'
+    url = '{}:{}/_get_log_head'.format(HOST, PORT)
-    spinner.text = f'Trying to connect to {url}'
+    spinner.text = 'Trying to connect to {}'.format(url)
     try:
         r = requests.get(url)
     except requests.exceptions.ConnectionError:
-        return (False, f'Can\'t connect to {url}')
+        return (False, 'Can\'t connect to {}').format(url)
     return (
         r.status_code == 200,
-        f'{url} {"not " if r.status_code != 200 else ""}reached. Status code [{r.status_code}]'
+        '{} {}reached. Status code [{}]'.format(url, "not " if r.status_code != 200 else "", r.status_code)
      )
 
 
@@ -389,14 +389,14 @@ def check_server_listening(spinner):
 def check_server_dynamic_enpoint(spinner):
     sleep_max = 15
     start_time = time.time()
-    url = f'{HOST}:{PORT}/_logs'
+    url = '{}:{}/_logs'.format(HOST, PORT)
     p = subprocess.Popen(
         ['curl', '-sfN', '--header', 'Accept: text/event-stream', url],
         stdout=subprocess.PIPE,
         bufsize=1)
     signal.alarm(sleep_max)
     return_flag = False
-    return_text = f'Dynamic endpoint returned data but not in the correct format.'
+    return_text = 'Dynamic endpoint returned data but not in the correct format.'
     try:
         for line in iter(p.stdout.readline, b''):
             if line.startswith(b'data: '):
@@ -404,15 +404,15 @@ def check_server_dynamic_enpoint(spinner):
                 try:
                     j = json.loads(data)
                     return_flag = True
-                    return_text = f'Dynamic endpoint returned data (took {time.time()-start_time:.2f}s)'
+                    return_text = 'Dynamic endpoint returned data (took {:.2f}s)'.format(time.time()-start_time)
                     signal.alarm(0)
                     break
                 except Exception as e:
                     return_flag = False
-                    return_text = f'Something went wrong. Output {line}'
+                    return_text = 'Something went wrong. Output {}'.format(line)
                     break
     except diagnostic_util.TimeoutException:
-        return_text = f'Dynamic endpoint did not returned data in the given time ({int(time.time()-start_time)}sec)'
+        return_text = 'Dynamic endpoint did not returned data in the given time ({}sec)'.format(int(time.time()-start_time))
     return (return_flag, return_text)
 
 

helpers/contributor_helper.py

@@ -100,7 +100,7 @@ class Contributor_helper:
                 self.DICO_PNTS_REWARD[categ] = self.default_pnts_per_contribution
 
         self.rankMultiplier = self.cfg_org_rank.getfloat('monthlyRanking' ,'rankMultiplier')
-        self.levelMax = self.cfg_org_rank.getint('monthlyRanking' ,'levelMax')
+        self.levelMax = self.cfg_org_rank.getint('monthlyRanking', 'levelMax')
 
         # REDIS KEYS
         self.keyDay         = KEYDAY
@@ -111,7 +111,6 @@ class Contributor_helper:
         self.keyTrophy      = "CONTRIB_TROPHY"
         self.keyLastAward   = "CONTRIB_LAST_AWARDS"
 
-
     ''' HELPER '''
     def getOrgLogoFromMISP(self, org):
         return "{}/img/orgs/{}.png".format(self.misp_web_url, org)
@@ -123,7 +122,7 @@ class Contributor_helper:
         self.logger.debug('Added to redis: keyname={}, org={}, count={}'.format(keyname, org, count))
 
     def publish_log(self, zmq_name, name, content, channel=""):
-        to_send = { 'name': name, 'log': json.dumps(content), 'zmqName': zmq_name }
+        to_send = {'name': name, 'log': json.dumps(content), 'zmqName': zmq_name }
         self.serv_log.publish(channel, json.dumps(to_send))
         self.logger.debug('Published: {}'.format(json.dumps(to_send)))
 

helpers/geo_helper.py

@@ -21,6 +21,7 @@ from phonenumbers import geocoder
 class InvalidCoordinate(Exception):
     pass
 
+
 class Geo_helper:
     def __init__(self, serv_redis_db, cfg):
         self.serv_redis_db = serv_redis_db
@@ -62,7 +63,12 @@ class Geo_helper:
             print(error)
             print("Please fix the above and try again.")
             sys.exit(126)
-        self.country_to_iso = { country.name: country.alpha_2 for country in pycountry.countries}
+        self.country_to_iso = {}
+        for country in pycountry.countries:
+            try:
+                self.country_to_iso[country.name] = country.alpha_2
+            except AttributeError:
+                pass
         with open(self.PATH_TO_JSON) as f:
             self.country_code_to_coord = json.load(f)
 
@@ -208,9 +214,10 @@ class Geo_helper:
             print("Please fix the above, and make sure you use a redis version that supports the GEOADD command.")
             print("To test for support: echo \"help GEOADD\"| redis-cli")
         self.logger.debug('Added to redis: keyname={}, lon={}, lat={}, content={}'.format(keyname, lon, lat, content))
+
     def push_to_redis_zset(self, keyCateg, toAdd, endSubkey="", count=1):
         if not isinstance(toAdd, str):
-            self.logger.warning(f'Can\'t add to redis, element is not of type String. {type(toAdd)}')
+            self.logger.warning('Can\'t add to redis, element is not of type String. {}'.format(type(toAdd)))
             return
         now = datetime.datetime.now()
         today_str = util.getDateStrFormat(now)

start_all.sh

@@ -6,6 +6,29 @@ GREEN="\\033[1;32m"
 DEFAULT="\\033[0;39m"
 RED="\\033[1;31m"
 
+function wait_until_redis_is_ready {
+    redis_not_ready=true
+    while $redis_not_ready; do
+        if checking_redis; then
+            redis_not_ready=false;
+        else
+            sleep 1
+        fi
+    done
+    echo -e $GREEN"* Redis 6250 is running"$DEFAULT
+}
+
+function checking_redis {
+    flag_redis=0
+    bash -c 'redis-cli -p 6250 PING | grep "PONG" &> /dev/null'
+    if [ ! $? == 0 ]; then
+        echo -e $RED"Redis 6250 not ready"$DEFAULT
+        flag_redis=1
+    fi
+    sleep 0.1
+    return $flag_redis;
+}
+
 # Getting CWD where bash script resides
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 DASH_HOME="${DIR}"
@@ -61,6 +84,8 @@ else
 fi
 
 sleep 0.1
+wait_until_redis_is_ready;
+
 if [ "${check_dashboard_port}" == "1" ]; then
     echo -e $GREEN"\t* Launching flask server"$DEFAULT
     ${ENV_PY} ./server.py &

start_zmq.sh

@@ -1,5 +1,4 @@
 #!/usr/bin/env bash
-
 #set -x
 
 GREEN="\\033[1;32m"
@@ -9,6 +8,7 @@ RED="\\033[1;31m"
 # Getting CWD where bash script resides
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 DASH_HOME="${DIR}"
+SCREEN_NAME="Misp_Dashboard"
 
 cd ${DASH_HOME}
 
@@ -20,11 +20,19 @@ else
     exit 1
 fi
 
+PID_SCREEN=$(screen -ls | grep ${SCREEN_NAME} | cut -f2 | cut -d. -f1)
+if [[ $PID_SCREEN ]]; then
+    echo -e $RED"* A screen '$SCREEN_NAME' is already launched"$DEFAULT
+    echo -e $GREEN"Killing $PID_SCREEN"$DEFAULT;
+    kill $PID_SCREEN
+else
+    echo 'No screen detected'
+fi
+
+screen -dmS ${SCREEN_NAME}
+
 ps auxw |grep zmq_subscriber.py |grep -v grep ; check_zmq_subscriber=$?
 ps auxw |grep zmq_dispatcher.py |grep -v grep ; check_zmq_dispatcher=$?
-
-screen -dmS "Misp_Dashboard"
-
 sleep 0.1
 if [ "${check_zmq_subscriber}" == "1" ]; then
     echo -e $GREEN"\t* Launching zmq subscribers"$DEFAULT
@@ -39,4 +47,4 @@ if [ "${check_zmq_dispatcher}" == "1" ]; then
     screen -S "Misp_Dashboard" -X screen -t "zmq-dispacher" bash -c ${ENV_PY}' ./zmq_dispatcher.py; read x'
 else
     echo -e $RED"\t* NOT starting zmq dispatcher, made a rather unrealiable ps -auxw | grep for zmq_dispatcher.py, and something seems to be there… please double check if this is good!"$DEFAULT
-fi
+fi

static/js/contrib.js

@@ -152,6 +152,7 @@ function getMonthlyRankIcon(rank, size, header) {
             img.width = size;
         }
     }
+    img.setAttribute('onerror', "this.style.display='none'");
     return img.outerHTML;
 }
 
@@ -167,7 +168,8 @@ function getOrgRankIcon(rank, size) {
     obj.src = rankLogoPath;
     obj.type = "image/svg"
     obj.title = org_rank_obj[rank];
-    obj.classList.add('orgRankClass')
+    obj.classList.add('orgRankClass');
+    obj.setAttribute('onerror', "this.style.display='none'");
     return obj.outerHTML;
 }
 
@@ -177,8 +179,9 @@ function createImg(source, size) {
     obj.width = size;
     obj.style.margin = 'auto';
     obj.src = source;
-    obj.type = "image/png"
+    obj.type = "image/png";
-    obj.alt = ""
+    obj.alt = "";
+    obj.setAttribute('onerror', "this.style.display='none'");
     return obj.outerHTML;
 }
 
@@ -187,10 +190,11 @@ function createTrophyImg(rank, size, categ) {
     obj.height = size;
     obj.width = size;
     obj.style.margin = 'auto';
-    obj.src = url_baseTrophyLogo+rank+'.png';;
+    obj.src = url_baseTrophyLogo+rank+'.png';
     obj.title = trophy_title[rank] + " in " + categ;
-    obj.type = "image/png"
+    obj.type = "image/png";
-    obj.alt = ""
+    obj.alt = "";
+    obj.setAttribute('onerror', "this.style.display='none'");
     return obj.outerHTML;
 }
 
@@ -208,6 +212,7 @@ function createHonorImg(array, size) {
         obj.style.margin = 'auto';
         obj.title = org_honor_badge_title[badgeNum];
         obj.src = url_baseHonorLogo+badgeNum+'.svg';
+        obj.setAttribute('onerror', "this.style.display='none'");
         div.appendChild(obj);
     }
     div.style.width = 32*array.length+'px';
@@ -563,7 +568,7 @@ function generate_table_ranking_on_category(categ) {
             var rank = arr[2];
             var tr = $('<tr></tr>');
             tr.append($('<td style="width: 100px;">'+i+'</td>'));
-            tr.append($('<td style="width: 100px;"><img src="'+url_baseTrophyLogo+rank+'.png" width="30" height="30"></td>'));
+            tr.append($('<td style="width: 100px;"><img src="'+url_baseTrophyLogo+rank+'.png" width="30" height="30"  onerror="this.style.display=\'none\'"></td>'));
             tr.append($('<td style="width: 200px;">'+points+'</td>'));
             tr.append($('<td><a href="?org='+org+'">'+org+'</a></td>'));
             if (currOrg == org) {

static/js/index/index.js

@@ -445,6 +445,13 @@ function createHead(callback) {
                 }
             },
 
+            changeOptions: function(options) {
+                var that = this;
+                Object.keys(options).forEach(function (optionName) {
+                    that._options[optionName] = options[optionName];
+                });
+            },
+
             fetch_predata: function() {
                 var that = this;
                 if (this._options.preDataURL !== null) {
@@ -808,10 +815,14 @@ $(document).ready(function() {
             $panel.detach().prependTo('#page-wrapper')
             $panel.addClass('liveLogFullScreen');
             $this.data('isfullscreen', true);
+            $panel.find('#divLogTable').css({'overflow-y': 'auto'});
+            livelog.changeOptions({tableMaxEntries: 300});
         } else {
             $panel.detach().appendTo('#rightCol')
             $panel.removeClass('liveLogFullScreen');
             $this.data('isfullscreen', false);
+            $panel.find('#divLogTable').css({'overflow': 'hidden'});
+            livelog.changeOptions({tableMaxEntries: 50});
         }
     });
 

templates/contrib.html

@@ -120,7 +120,7 @@
                                 <tbody id='bodyTablerankingModal'>
                                       {% for item in org_rank_list %}
                                           <tr data-rank={{ loop.index }}>
-                                              <td style='padding: 0px; text-align: right;'><img height='35px' width='70px' style="margin-right: 20px;" src="{{ url_for('static', filename='pics/rankingMISPOrg/1.svg')[:-5]}}{{ item[0] }}.svg" type='image/svg' style="margin: auto;"</img></td>
+                                              <td style='padding: 0px; text-align: right;'><img height='35px' width='70px' style="margin-right: 20px;" src="{{ url_for('static', filename='pics/rankingMISPOrg/1.svg')[:-5]}}{{ item[0] }}.svg" type='image/svg' style="margin: auto;" onerror="this.style.display='none'"</img></td>
                                               <td>{{ item[1] }}</td>
                                               <td>{{ item[2] }}</td>
                                               <td>{{ item[3] }}</td>
@@ -172,7 +172,7 @@
                                     <tr>
                                         <td>
                                             <div id="divBadge_{{ loop.index }}" class="circleBadgeSmall circlBadgeNotAcquired">
-                                                <img height='48px' width='48' class="" style="margin-top: 3px;" src="{{ url_for('static', filename='pics/MISPHonorableIcons/1.svg')[:-5]}}{{ item[0] }}.svg" type='image/svg' style="margin: auto;"</img>
+                                                <img height='48px' width='48' class="" style="margin-top: 3px;" src="{{ url_for('static', filename='pics/MISPHonorableIcons/1.svg')[:-5]}}{{ item[0] }}.svg" type='image/svg' style="margin: auto;" onerror="this.style.display='none'"</img>
                                             </div>
                                         </td>
                                         <td style="padding-left: 15px;">{{ item[1] }}</td>

templates/index.html

@@ -188,11 +188,15 @@ div.dataTables_scrollHead table.dataTable {
     top: 66px !important;
     left: 15px !important;
     right: 10px !important;
-    z-index: 1001 !important;
+    z-index: 990 !important;
     bottom: -7px !important;
     height: unset !important;
 }
 
+div.leaflet-bottom {
+    z-index: 900;
+}
+
 .rowTableIsObject {
     position: absolute;
     right: 15px;

updates.py

@@ -65,9 +65,9 @@ def exec_updates(db_version):
 
     if result:
         serv_redis_db.set(cfg.get('RedisDB', 'dbVersion'), db_version)
-        update_logger.warning(f'dbVersion sets to {db_version}')
+        update_logger.warning('dbVersion sets to {}'.format(db_version))
     else:
-        update_logger.error(f'Something went wrong. {result}')
+        update_logger.error('Something went wrong. {}'.format(result))
 
 
 # Data format changed. Wipe the key.