Merge pull request #121 from mokaddem/fewFixes2

Various fixes and improvements
pull/135/head v1.3
Sami Mokaddem 2019-08-30 13:16:47 +02:00 committed by GitHub
commit 0ac7e7cf84
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 110 additions and 51 deletions

View File

@ -18,7 +18,7 @@ try:
from halo import Halo
except ModuleNotFoundError as e:
print('Dependency not met. Either not in a virtualenv or dependency not installed.')
print(f'- Error: {e}')
print('- Error: {}'.format(e))
sys.exit(1)
'''
@ -79,12 +79,12 @@ def add_spinner(_func=None, name='dots'):
else:
status = False
flag_skip = True
spinner.fail(f'{human_func_name} - Function return unexpected result: {str(result)}')
spinner.fail('{} - Function return unexpected result: {}'.format(human_func_name, str(result)))
if not flag_skip:
text = human_func_result
if output is not None and len(output) > 0:
text += f': {output}'
text += ': {}'.format(output)
if isinstance(status, bool) and status:
spinner.succeed(text)
@ -111,8 +111,8 @@ def check_virtual_environment_and_packages(spinner):
return (False, 'This diagnostic tool should be started inside a virtual environment.')
else:
if redis.__version__.startswith('2'):
return (False, f'''Redis python client have version {redis.__version__}. Version 3.x required.
\t [inside virtualenv] pip3 install -U redis''')
return (False, '''Redis python client have version {}. Version 3.x required.
\t [inside virtualenv] pip3 install -U redis'''.format(redis.__version__))
else:
return (True, '')
@ -139,7 +139,7 @@ def check_configuration(spinner):
return_text = '''Configuration incomplete.
\tUpdate your configuration file `config.cfg`.\n\t Faulty fields:\n'''
for field_name in faulties:
return_text += f'\t\t- {field_name}\n'
return_text += '\t\t- {}\n'.format(field_name)
return (False, return_text)
@ -192,7 +192,7 @@ def check_zmq(spinner):
flag_skip = True
break
else:
spinner.text = f'checking zmq of {misp_instance.get("name")} - elapsed time: {int(time.time() - start_time)}s'
spinner.text = 'checking zmq of {} - elapsed time: {}s'.format(misp_instance.get("name"), int(time.time() - start_time))
if not flag_skip:
instances_status[misp_instance.get('name')] = False
@ -202,7 +202,7 @@ def check_zmq(spinner):
elif any(results):
return_text = 'Connection to ZMQ stream(s) failed.\n'
for name, status in instances_status.items():
return_text += f'\t{name}: {"success" if status else "failed"}\n'
return_text += '\t{}: {}\n'.format(name, "success" if status else "failed")
return (True, return_text)
else:
return (False, '''Can\'t connect to the ZMQ stream(s).
@ -257,14 +257,14 @@ def check_subscriber_status(spinner):
target = split[4]
except IndexError:
pass
if action == '"LPUSH"' and target == f'\"{configuration_file.get("RedisLIST", "listName")}\"':
if action == '"LPUSH"' and target == '\"{}\"'.format(configuration_file.get("RedisLIST", "listName")):
signal.alarm(0)
break
else:
spinner.text = f'Checking subscriber status - elapsed time: {int(time.time() - start_time)}s'
spinner.text = 'Checking subscriber status - elapsed time: {}s'.format(int(time.time() - start_time))
except diagnostic_util.TimeoutException:
return_text = f'''zmq_subscriber seems not to be working.
\t Consider restarting it: {pgrep_subscriber_output}'''
return_text = '''zmq_subscriber seems not to be working.
\t Consider restarting it: {}'''.format(pgrep_subscriber_output)
return (False, return_text)
return (True, 'subscriber is running and populating the buffer')
@ -278,7 +278,7 @@ def check_buffer_queue(spinner):
warning_threshold = 100
elements_in_list = redis_server.llen(configuration_file.get('RedisLIST', 'listName'))
return_status = 'warning' if elements_in_list > warning_threshold else ('info' if elements_in_list > 0 else True)
return_text = f'Currently {elements_in_list} items in the buffer'
return_text = 'Currently {} items in the buffer'.format(elements_in_list)
return (return_status, return_text)
@ -308,8 +308,8 @@ def check_buffer_change_rate(spinner):
if next_refresh < time_slept:
next_refresh = time_slept + refresh_frequency
change_rate_text = f'{change_increase}/sec\t{change_decrease}/sec'
spinner.text = f'Buffer: {elements_in_list}\t{change_rate_text}'
change_rate_text = '{}/sec\t{}/sec'.format(change_increase, change_decrease)
spinner.text = 'Buffer: {}\t{}'.format(elements_in_list, change_rate_text)
if consecutive_no_rate_change == 3:
time_slept = sleep_max
@ -322,7 +322,7 @@ def check_buffer_change_rate(spinner):
if time_slept >= sleep_max:
return_flag = elements_in_list == 0 or (elements_in_list < elements_in_inlist_init or elements_in_list < 2)
return_text = f'Buffer is consumed {"faster" if return_flag else "slower" } than being populated'
return_text = 'Buffer is consumed {} than being populated'.format("faster" if return_flag else "slower")
break
time.sleep(sleep_duration)
@ -354,18 +354,18 @@ def check_dispatcher_status(spinner):
if reply is None:
if time_slept >= sleep_max:
return_flag = False
return_text = f'zmq_dispatcher did not respond in the given time ({int(sleep_max)}s)'
return_text = 'zmq_dispatcher did not respond in the given time ({}s)'.format(int(sleep_max))
if len(pgrep_dispatcher_output) > 0:
return_text += f'\n\t➥ Consider restarting it: {pgrep_dispatcher_output}'
return_text += '\n\t➥ Consider restarting it: {}'.format(pgrep_dispatcher_output)
else:
return_text += '\n\t➥ Consider starting it'
break
time.sleep(sleep_duration)
spinner.text = f'Dispatcher status: No response yet'
spinner.text = 'Dispatcher status: No response yet'
time_slept += sleep_duration
else:
return_flag = True
return_text = f'Took {float(reply):.2f}s to complete'
return_text = 'Took {:.2f}s to complete'.format(float(reply))
break
return (return_flag, return_text)
@ -373,15 +373,15 @@ def check_dispatcher_status(spinner):
@add_spinner
def check_server_listening(spinner):
url = f'{HOST}:{PORT}/_get_log_head'
spinner.text = f'Trying to connect to {url}'
url = '{}:{}/_get_log_head'.format(HOST, PORT)
spinner.text = 'Trying to connect to {}'.format(url)
try:
r = requests.get(url)
except requests.exceptions.ConnectionError:
return (False, f'Can\'t connect to {url}')
return (False, 'Can\'t connect to {}').format(url)
return (
r.status_code == 200,
f'{url} {"not " if r.status_code != 200 else ""}reached. Status code [{r.status_code}]'
'{} {}reached. Status code [{}]'.format(url, "not " if r.status_code != 200 else "", r.status_code)
)
@ -389,14 +389,14 @@ def check_server_listening(spinner):
def check_server_dynamic_enpoint(spinner):
sleep_max = 15
start_time = time.time()
url = f'{HOST}:{PORT}/_logs'
url = '{}:{}/_logs'.format(HOST, PORT)
p = subprocess.Popen(
['curl', '-sfN', '--header', 'Accept: text/event-stream', url],
stdout=subprocess.PIPE,
bufsize=1)
signal.alarm(sleep_max)
return_flag = False
return_text = f'Dynamic endpoint returned data but not in the correct format.'
return_text = 'Dynamic endpoint returned data but not in the correct format.'
try:
for line in iter(p.stdout.readline, b''):
if line.startswith(b'data: '):
@ -404,15 +404,15 @@ def check_server_dynamic_enpoint(spinner):
try:
j = json.loads(data)
return_flag = True
return_text = f'Dynamic endpoint returned data (took {time.time()-start_time:.2f}s)'
return_text = 'Dynamic endpoint returned data (took {:.2f}s)'.format(time.time()-start_time)
signal.alarm(0)
break
except Exception as e:
return_flag = False
return_text = f'Something went wrong. Output {line}'
return_text = 'Something went wrong. Output {}'.format(line)
break
except diagnostic_util.TimeoutException:
return_text = f'Dynamic endpoint did not returned data in the given time ({int(time.time()-start_time)}sec)'
return_text = 'Dynamic endpoint did not returned data in the given time ({}sec)'.format(int(time.time()-start_time))
return (return_flag, return_text)

View File

@ -100,7 +100,7 @@ class Contributor_helper:
self.DICO_PNTS_REWARD[categ] = self.default_pnts_per_contribution
self.rankMultiplier = self.cfg_org_rank.getfloat('monthlyRanking' ,'rankMultiplier')
self.levelMax = self.cfg_org_rank.getint('monthlyRanking' ,'levelMax')
self.levelMax = self.cfg_org_rank.getint('monthlyRanking', 'levelMax')
# REDIS KEYS
self.keyDay = KEYDAY
@ -111,7 +111,6 @@ class Contributor_helper:
self.keyTrophy = "CONTRIB_TROPHY"
self.keyLastAward = "CONTRIB_LAST_AWARDS"
''' HELPER '''
def getOrgLogoFromMISP(self, org):
return "{}/img/orgs/{}.png".format(self.misp_web_url, org)
@ -123,7 +122,7 @@ class Contributor_helper:
self.logger.debug('Added to redis: keyname={}, org={}, count={}'.format(keyname, org, count))
def publish_log(self, zmq_name, name, content, channel=""):
to_send = { 'name': name, 'log': json.dumps(content), 'zmqName': zmq_name }
to_send = {'name': name, 'log': json.dumps(content), 'zmqName': zmq_name }
self.serv_log.publish(channel, json.dumps(to_send))
self.logger.debug('Published: {}'.format(json.dumps(to_send)))

View File

@ -21,6 +21,7 @@ from phonenumbers import geocoder
class InvalidCoordinate(Exception):
pass
class Geo_helper:
def __init__(self, serv_redis_db, cfg):
self.serv_redis_db = serv_redis_db
@ -62,7 +63,12 @@ class Geo_helper:
print(error)
print("Please fix the above and try again.")
sys.exit(126)
self.country_to_iso = { country.name: country.alpha_2 for country in pycountry.countries}
self.country_to_iso = {}
for country in pycountry.countries:
try:
self.country_to_iso[country.name] = country.alpha_2
except AttributeError:
pass
with open(self.PATH_TO_JSON) as f:
self.country_code_to_coord = json.load(f)
@ -208,9 +214,10 @@ class Geo_helper:
print("Please fix the above, and make sure you use a redis version that supports the GEOADD command.")
print("To test for support: echo \"help GEOADD\"| redis-cli")
self.logger.debug('Added to redis: keyname={}, lon={}, lat={}, content={}'.format(keyname, lon, lat, content))
def push_to_redis_zset(self, keyCateg, toAdd, endSubkey="", count=1):
if not isinstance(toAdd, str):
self.logger.warning(f'Can\'t add to redis, element is not of type String. {type(toAdd)}')
self.logger.warning('Can\'t add to redis, element is not of type String. {}'.format(type(toAdd)))
return
now = datetime.datetime.now()
today_str = util.getDateStrFormat(now)

View File

@ -6,6 +6,29 @@ GREEN="\\033[1;32m"
DEFAULT="\\033[0;39m"
RED="\\033[1;31m"
function wait_until_redis_is_ready {
redis_not_ready=true
while $redis_not_ready; do
if checking_redis; then
redis_not_ready=false;
else
sleep 1
fi
done
echo -e $GREEN"* Redis 6250 is running"$DEFAULT
}
function checking_redis {
flag_redis=0
bash -c 'redis-cli -p 6250 PING | grep "PONG" &> /dev/null'
if [ ! $? == 0 ]; then
echo -e $RED"Redis 6250 not ready"$DEFAULT
flag_redis=1
fi
sleep 0.1
return $flag_redis;
}
# Getting CWD where bash script resides
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
DASH_HOME="${DIR}"
@ -61,6 +84,8 @@ else
fi
sleep 0.1
wait_until_redis_is_ready;
if [ "${check_dashboard_port}" == "1" ]; then
echo -e $GREEN"\t* Launching flask server"$DEFAULT
${ENV_PY} ./server.py &

View File

@ -1,5 +1,4 @@
#!/usr/bin/env bash
#set -x
GREEN="\\033[1;32m"
@ -9,6 +8,7 @@ RED="\\033[1;31m"
# Getting CWD where bash script resides
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
DASH_HOME="${DIR}"
SCREEN_NAME="Misp_Dashboard"
cd ${DASH_HOME}
@ -20,11 +20,19 @@ else
exit 1
fi
PID_SCREEN=$(screen -ls | grep ${SCREEN_NAME} | cut -f2 | cut -d. -f1)
if [[ $PID_SCREEN ]]; then
echo -e $RED"* A screen '$SCREEN_NAME' is already launched"$DEFAULT
echo -e $GREEN"Killing $PID_SCREEN"$DEFAULT;
kill $PID_SCREEN
else
echo 'No screen detected'
fi
screen -dmS ${SCREEN_NAME}
ps auxw |grep zmq_subscriber.py |grep -v grep ; check_zmq_subscriber=$?
ps auxw |grep zmq_dispatcher.py |grep -v grep ; check_zmq_dispatcher=$?
screen -dmS "Misp_Dashboard"
sleep 0.1
if [ "${check_zmq_subscriber}" == "1" ]; then
echo -e $GREEN"\t* Launching zmq subscribers"$DEFAULT
@ -39,4 +47,4 @@ if [ "${check_zmq_dispatcher}" == "1" ]; then
screen -S "Misp_Dashboard" -X screen -t "zmq-dispacher" bash -c ${ENV_PY}' ./zmq_dispatcher.py; read x'
else
echo -e $RED"\t* NOT starting zmq dispatcher, made a rather unrealiable ps -auxw | grep for zmq_dispatcher.py, and something seems to be there… please double check if this is good!"$DEFAULT
fi
fi

View File

@ -152,6 +152,7 @@ function getMonthlyRankIcon(rank, size, header) {
img.width = size;
}
}
img.setAttribute('onerror', "this.style.display='none'");
return img.outerHTML;
}
@ -167,7 +168,8 @@ function getOrgRankIcon(rank, size) {
obj.src = rankLogoPath;
obj.type = "image/svg"
obj.title = org_rank_obj[rank];
obj.classList.add('orgRankClass')
obj.classList.add('orgRankClass');
obj.setAttribute('onerror', "this.style.display='none'");
return obj.outerHTML;
}
@ -177,8 +179,9 @@ function createImg(source, size) {
obj.width = size;
obj.style.margin = 'auto';
obj.src = source;
obj.type = "image/png"
obj.alt = ""
obj.type = "image/png";
obj.alt = "";
obj.setAttribute('onerror', "this.style.display='none'");
return obj.outerHTML;
}
@ -187,10 +190,11 @@ function createTrophyImg(rank, size, categ) {
obj.height = size;
obj.width = size;
obj.style.margin = 'auto';
obj.src = url_baseTrophyLogo+rank+'.png';;
obj.src = url_baseTrophyLogo+rank+'.png';
obj.title = trophy_title[rank] + " in " + categ;
obj.type = "image/png"
obj.alt = ""
obj.type = "image/png";
obj.alt = "";
obj.setAttribute('onerror', "this.style.display='none'");
return obj.outerHTML;
}
@ -208,6 +212,7 @@ function createHonorImg(array, size) {
obj.style.margin = 'auto';
obj.title = org_honor_badge_title[badgeNum];
obj.src = url_baseHonorLogo+badgeNum+'.svg';
obj.setAttribute('onerror', "this.style.display='none'");
div.appendChild(obj);
}
div.style.width = 32*array.length+'px';
@ -563,7 +568,7 @@ function generate_table_ranking_on_category(categ) {
var rank = arr[2];
var tr = $('<tr></tr>');
tr.append($('<td style="width: 100px;">'+i+'</td>'));
tr.append($('<td style="width: 100px;"><img src="'+url_baseTrophyLogo+rank+'.png" width="30" height="30"></td>'));
tr.append($('<td style="width: 100px;"><img src="'+url_baseTrophyLogo+rank+'.png" width="30" height="30" onerror="this.style.display=\'none\'"></td>'));
tr.append($('<td style="width: 200px;">'+points+'</td>'));
tr.append($('<td><a href="?org='+org+'">'+org+'</a></td>'));
if (currOrg == org) {

View File

@ -445,6 +445,13 @@ function createHead(callback) {
}
},
changeOptions: function(options) {
var that = this;
Object.keys(options).forEach(function (optionName) {
that._options[optionName] = options[optionName];
});
},
fetch_predata: function() {
var that = this;
if (this._options.preDataURL !== null) {
@ -808,10 +815,14 @@ $(document).ready(function() {
$panel.detach().prependTo('#page-wrapper')
$panel.addClass('liveLogFullScreen');
$this.data('isfullscreen', true);
$panel.find('#divLogTable').css({'overflow-y': 'auto'});
livelog.changeOptions({tableMaxEntries: 300});
} else {
$panel.detach().appendTo('#rightCol')
$panel.removeClass('liveLogFullScreen');
$this.data('isfullscreen', false);
$panel.find('#divLogTable').css({'overflow': 'hidden'});
livelog.changeOptions({tableMaxEntries: 50});
}
});

View File

@ -120,7 +120,7 @@
<tbody id='bodyTablerankingModal'>
{% for item in org_rank_list %}
<tr data-rank={{ loop.index }}>
<td style='padding: 0px; text-align: right;'><img height='35px' width='70px' style="margin-right: 20px;" src="{{ url_for('static', filename='pics/rankingMISPOrg/1.svg')[:-5]}}{{ item[0] }}.svg" type='image/svg' style="margin: auto;"</img></td>
<td style='padding: 0px; text-align: right;'><img height='35px' width='70px' style="margin-right: 20px;" src="{{ url_for('static', filename='pics/rankingMISPOrg/1.svg')[:-5]}}{{ item[0] }}.svg" type='image/svg' style="margin: auto;" onerror="this.style.display='none'"</img></td>
<td>{{ item[1] }}</td>
<td>{{ item[2] }}</td>
<td>{{ item[3] }}</td>
@ -172,7 +172,7 @@
<tr>
<td>
<div id="divBadge_{{ loop.index }}" class="circleBadgeSmall circlBadgeNotAcquired">
<img height='48px' width='48' class="" style="margin-top: 3px;" src="{{ url_for('static', filename='pics/MISPHonorableIcons/1.svg')[:-5]}}{{ item[0] }}.svg" type='image/svg' style="margin: auto;"</img>
<img height='48px' width='48' class="" style="margin-top: 3px;" src="{{ url_for('static', filename='pics/MISPHonorableIcons/1.svg')[:-5]}}{{ item[0] }}.svg" type='image/svg' style="margin: auto;" onerror="this.style.display='none'"</img>
</div>
</td>
<td style="padding-left: 15px;">{{ item[1] }}</td>

View File

@ -188,11 +188,15 @@ div.dataTables_scrollHead table.dataTable {
top: 66px !important;
left: 15px !important;
right: 10px !important;
z-index: 1001 !important;
z-index: 990 !important;
bottom: -7px !important;
height: unset !important;
}
div.leaflet-bottom {
z-index: 900;
}
.rowTableIsObject {
position: absolute;
right: 15px;

View File

@ -65,9 +65,9 @@ def exec_updates(db_version):
if result:
serv_redis_db.set(cfg.get('RedisDB', 'dbVersion'), db_version)
update_logger.warning(f'dbVersion sets to {db_version}')
update_logger.warning('dbVersion sets to {}'.format(db_version))
else:
update_logger.error(f'Something went wrong. {result}')
update_logger.error('Something went wrong. {}'.format(result))
# Data format changed. Wipe the key.