From 6e225f8778d4af8d2b5b8937e59662f006cc7dbe Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Sat, 31 Mar 2018 18:55:16 +0900 Subject: [PATCH 01/18] - Brain fart, oversaw (read didn't look for) start_all.sh --- run.sh | 6 ------ 1 file changed, 6 deletions(-) delete mode 100755 run.sh diff --git a/run.sh b/run.sh deleted file mode 100755 index 0479133..0000000 --- a/run.sh +++ /dev/null @@ -1,6 +0,0 @@ -. ./DASHENV/bin/activate -./zmq_subscriber.py & -./zmq_dispatcher.py & -export FLASK_DEBUG=1 -export FLASK_APP=server.py -flask run --host=0.0.0.0 --port=8001 From a6c4f2928dc65bc3e113f1c7182f0474dcb8ff8f Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Sat, 31 Mar 2018 19:21:52 +0900 Subject: [PATCH 02/18] - Fixed the brokeness of: - echo export DASH_CONFIG=$(pwd)/config/ >> ./DASHENV/bin/activate -- This is mostly wrong for people not running misp-dashboard interactively. -- It also assumes people use DASHEN as their virtualenv name (One more issue along those lines remain, $DASH_HOME) --- give_honors_to_org.py | 2 +- helpers/contributor_helper.py | 2 +- install_dependencies.sh | 1 - misp-dashboard.wsgi | 2 +- server.py | 2 +- zmq_dispatcher.py | 2 +- zmq_subscriber.py | 2 +- 7 files changed, 6 insertions(+), 7 deletions(-) diff --git a/give_honors_to_org.py b/give_honors_to_org.py index edf4bbc..82f2a30 100755 --- a/give_honors_to_org.py +++ b/give_honors_to_org.py @@ -9,7 +9,7 @@ import util import contributor_helper ONE_DAY = 60*60*24 -configfile = os.path.join(os.environ['DASH_CONFIG'], 'config.cfg') +configfile = os.path.join(os.path.dirname(os.path.realpath(_file_)), 'config/config.cfg') cfg = configparser.ConfigParser() cfg.read(configfile) serv_log = redis.StrictRedis( diff --git a/helpers/contributor_helper.py b/helpers/contributor_helper.py index 10dac9a..19e00e1 100644 --- a/helpers/contributor_helper.py +++ b/helpers/contributor_helper.py @@ -23,7 +23,7 @@ class Contributor_helper: db=cfg.getint('RedisLog', 'db')) self.cfg = cfg self.cfg_org_rank = configparser.ConfigParser() - self.cfg_org_rank.read(os.path.join(os.environ['DASH_CONFIG'], 'ranking.cfg')) + self.cfg_org_rank.read(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'config/ranking.cfg') self.CHANNEL_LASTAWARDS = cfg.get('RedisLog', 'channelLastAwards') self.CHANNEL_LASTCONTRIB = cfg.get('RedisLog', 'channelLastContributor') self.users_helper = users_helper.Users_helper(serv_redis_db, cfg) diff --git a/install_dependencies.sh b/install_dependencies.sh index dc29094..c376f93 100755 --- a/install_dependencies.sh +++ b/install_dependencies.sh @@ -9,7 +9,6 @@ if [ -z "$VIRTUAL_ENV" ]; then virtualenv -p python3 DASHENV echo export DASH_HOME=$(pwd) >> ./DASHENV/bin/activate - echo export DASH_CONFIG=$(pwd)/config/ >> ./DASHENV/bin/activate . ./DASHENV/bin/activate fi diff --git a/misp-dashboard.wsgi b/misp-dashboard.wsgi index 47924c5..e71690a 100644 --- a/misp-dashboard.wsgi +++ b/misp-dashboard.wsgi @@ -1,4 +1,4 @@ import sys,os,os.path sys.path.insert(0, os.path.dirname(__file__)) -os.environ["DASH_CONFIG"] = os.path.join(os.path.dirname(__file__), "config") +os.environ["DASH_CONFIG"] = os.path.join(os.path.realpath(__file__), "config") from server import app as application diff --git a/server.py b/server.py index 02c9253..6166993 100755 --- a/server.py +++ b/server.py @@ -16,7 +16,7 @@ from helpers import contributor_helper from helpers import users_helper from helpers import trendings_helper -configfile = os.path.join(os.environ['DASH_CONFIG'], 'config.cfg') +configfile = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'config/config.cfg') cfg = configparser.ConfigParser() cfg.read(configfile) diff --git a/zmq_dispatcher.py b/zmq_dispatcher.py index 8441d25..bb54afe 100755 --- a/zmq_dispatcher.py +++ b/zmq_dispatcher.py @@ -18,7 +18,7 @@ from helpers import contributor_helper from helpers import users_helper from helpers import trendings_helper -configfile = os.path.join(os.environ['DASH_CONFIG'], 'config.cfg') +configfile = os.path.join(os.path.dirname(os.path.realpath(_file_)), 'config/config.cfg') cfg = configparser.ConfigParser() cfg.read(configfile) diff --git a/zmq_subscriber.py b/zmq_subscriber.py index 2a19866..f343892 100755 --- a/zmq_subscriber.py +++ b/zmq_subscriber.py @@ -10,7 +10,7 @@ import os import sys import json -configfile = os.path.join(os.environ['DASH_CONFIG'], 'config.cfg') +configfile = os.path.join(os.path.dirname(os.path.realpath(_file_)), 'config/config.cfg') cfg = configparser.ConfigParser() cfg.read(configfile) logDir = cfg.get('Log', 'directory') From b7be2afaa1d325504466f3e2f9509d62df1435e0 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Sat, 31 Mar 2018 19:36:17 +0900 Subject: [PATCH 03/18] - Small typo, one _ missing on each side ;) --- give_honors_to_org.py | 2 +- zmq_dispatcher.py | 2 +- zmq_subscriber.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/give_honors_to_org.py b/give_honors_to_org.py index 82f2a30..0878882 100755 --- a/give_honors_to_org.py +++ b/give_honors_to_org.py @@ -9,7 +9,7 @@ import util import contributor_helper ONE_DAY = 60*60*24 -configfile = os.path.join(os.path.dirname(os.path.realpath(_file_)), 'config/config.cfg') +configfile = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'config/config.cfg') cfg = configparser.ConfigParser() cfg.read(configfile) serv_log = redis.StrictRedis( diff --git a/zmq_dispatcher.py b/zmq_dispatcher.py index bb54afe..0f1f137 100755 --- a/zmq_dispatcher.py +++ b/zmq_dispatcher.py @@ -18,7 +18,7 @@ from helpers import contributor_helper from helpers import users_helper from helpers import trendings_helper -configfile = os.path.join(os.path.dirname(os.path.realpath(_file_)), 'config/config.cfg') +configfile = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'config/config.cfg') cfg = configparser.ConfigParser() cfg.read(configfile) diff --git a/zmq_subscriber.py b/zmq_subscriber.py index f343892..e8944c0 100755 --- a/zmq_subscriber.py +++ b/zmq_subscriber.py @@ -10,7 +10,7 @@ import os import sys import json -configfile = os.path.join(os.path.dirname(os.path.realpath(_file_)), 'config/config.cfg') +configfile = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'config/config.cfg') cfg = configparser.ConfigParser() cfg.read(configfile) logDir = cfg.get('Log', 'directory') From 1d57ce4525cb1bdbeda8451e41659d00ce7f468c Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Sat, 31 Mar 2018 19:40:03 +0900 Subject: [PATCH 04/18] - Another typo, closing parenthesis --- helpers/contributor_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/contributor_helper.py b/helpers/contributor_helper.py index 19e00e1..4a1a49a 100644 --- a/helpers/contributor_helper.py +++ b/helpers/contributor_helper.py @@ -23,7 +23,7 @@ class Contributor_helper: db=cfg.getint('RedisLog', 'db')) self.cfg = cfg self.cfg_org_rank = configparser.ConfigParser() - self.cfg_org_rank.read(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'config/ranking.cfg') + self.cfg_org_rank.read(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'config/ranking.cfg')) self.CHANNEL_LASTAWARDS = cfg.get('RedisLog', 'channelLastAwards') self.CHANNEL_LASTCONTRIB = cfg.get('RedisLog', 'channelLastContributor') self.users_helper = users_helper.Users_helper(serv_redis_db, cfg) From 4924da6ce5b3a5aae3ad96db4c3afb2116ff5a18 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Sat, 31 Mar 2018 19:43:50 +0900 Subject: [PATCH 05/18] - config directory is one up. --- helpers/contributor_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/contributor_helper.py b/helpers/contributor_helper.py index 4a1a49a..63b1fe4 100644 --- a/helpers/contributor_helper.py +++ b/helpers/contributor_helper.py @@ -23,7 +23,7 @@ class Contributor_helper: db=cfg.getint('RedisLog', 'db')) self.cfg = cfg self.cfg_org_rank = configparser.ConfigParser() - self.cfg_org_rank.read(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'config/ranking.cfg')) + self.cfg_org_rank.read(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../config/ranking.cfg')) self.CHANNEL_LASTAWARDS = cfg.get('RedisLog', 'channelLastAwards') self.CHANNEL_LASTCONTRIB = cfg.get('RedisLog', 'channelLastContributor') self.users_helper = users_helper.Users_helper(serv_redis_db, cfg) From a3ef669e801dda44328a83c85d7615bbcef8d990 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Sat, 31 Mar 2018 19:58:38 +0900 Subject: [PATCH 06/18] - Added very flaky check if something is running on port 6250 -- Mostly added this because the VM builder already launches a redis on port 6250 --- start_all.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/start_all.sh b/start_all.sh index de34af0..1831a61 100755 --- a/start_all.sh +++ b/start_all.sh @@ -15,6 +15,9 @@ if [ ! -e "${redis_dir}" ]; then redis_dir="" fi + +check_redis_port=$(netstat -an |grep LISTEN |grep 6250 |grep -v tcp6) + # Configure accordingly, remember: 0.0.0.0 exposes to every active IP interface, play safe and bind it to something you trust and know export FLASK_APP=server.py export FLASK_DEBUG=0 @@ -27,8 +30,12 @@ screenName="Misp-Dashboard" screen -dmS "$screenName" sleep 0.1 -echo -e $GREEN"\t* Launching Redis servers"$DEFAULT -screen -S "$screenName" -X screen -t "redis-server" bash -c $redis_dir'redis-server '$conf_dir'6250.conf && echo "Started Redis" ; read x' +if [ -z "${check_redis_port}" ]; then + echo -e $GREEN"\t* Launching Redis servers"$DEFAULT +# screen -S "$screenName" -X screen -t "redis-server" bash -c $redis_dir'redis-server '$conf_dir'6250.conf && echo "Started Redis" ; read x' +else + echo -e $RED"\t* NOT starting Redis server, made a very unrealiable check on port 6250, and something seems to be there… please double check if this is good!"$DEFAULT +fi echo -e $GREEN"\t* Launching zmq subscriber"$DEFAULT screen -S "$screenName" -X screen -t "zmq-subscriber" bash -c 'echo "Starting zmq-subscriber" ; ./zmq_subscriber.py; read x' From 9691536a21dcf2d894996f6bb478c399e2740fc0 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Mon, 2 Apr 2018 01:59:08 +0900 Subject: [PATCH 07/18] =?UTF-8?q?-=20Remove=20DASH=5FHOME=20legacy?= =?UTF-8?q?=E2=80=A6=20--=20Now=20more=20dynamic=20-=20Added=20python=20ve?= =?UTF-8?q?nv=20check=20-=20Removed=20necessity=20of=20activating=20the=20?= =?UTF-8?q?the=20venv?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- install_dependencies.sh | 2 -- start_all.sh | 20 ++++++++++++++------ tests/start_framework.sh | 3 ++- tests/terminate_framework.sh | 4 +++- tests/testall.sh | 4 +++- 5 files changed, 22 insertions(+), 11 deletions(-) diff --git a/install_dependencies.sh b/install_dependencies.sh index c376f93..e52300b 100755 --- a/install_dependencies.sh +++ b/install_dependencies.sh @@ -8,8 +8,6 @@ sudo apt-get install python3-virtualenv virtualenv screen redis-server unzip -y if [ -z "$VIRTUAL_ENV" ]; then virtualenv -p python3 DASHENV - echo export DASH_HOME=$(pwd) >> ./DASHENV/bin/activate - . ./DASHENV/bin/activate fi diff --git a/start_all.sh b/start_all.sh index 1831a61..6473228 100755 --- a/start_all.sh +++ b/start_all.sh @@ -7,7 +7,16 @@ GREEN="\\033[1;32m" DEFAULT="\\033[0;39m" RED="\\033[1;31m" -[ -z "$DASH_HOME" ] && echo "Needs the env var DASHENV. (Did you: . ./DASHENV/bin/activate ) Please run the script from the virtual environment." && exit 1; +# Getting CWD where bash script resides +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +if [ -e "${DIR}/DASHENV/bin/python" ]; then + echo "dashboard virtualenv seems to exist, good" + ENV_PY=${DIR}/DASHENV/bin/python +else + echo "Please make sure you have a dashboard environment, au revoir" + exit 1 +fi redis_dir="${DASH_HOME}/../redis/src/" if [ ! -e "${redis_dir}" ]; then @@ -15,7 +24,6 @@ if [ ! -e "${redis_dir}" ]; then redis_dir="" fi - check_redis_port=$(netstat -an |grep LISTEN |grep 6250 |grep -v tcp6) # Configure accordingly, remember: 0.0.0.0 exposes to every active IP interface, play safe and bind it to something you trust and know @@ -32,16 +40,16 @@ screen -dmS "$screenName" sleep 0.1 if [ -z "${check_redis_port}" ]; then echo -e $GREEN"\t* Launching Redis servers"$DEFAULT -# screen -S "$screenName" -X screen -t "redis-server" bash -c $redis_dir'redis-server '$conf_dir'6250.conf && echo "Started Redis" ; read x' + screen -S "$screenName" -X screen -t "redis-server" bash -c $redis_dir'redis-server '$conf_dir'6250.conf && echo "Started Redis" ; read x' else echo -e $RED"\t* NOT starting Redis server, made a very unrealiable check on port 6250, and something seems to be there… please double check if this is good!"$DEFAULT fi echo -e $GREEN"\t* Launching zmq subscriber"$DEFAULT -screen -S "$screenName" -X screen -t "zmq-subscriber" bash -c 'echo "Starting zmq-subscriber" ; ./zmq_subscriber.py; read x' +screen -S "$screenName" -X screen -t "zmq-subscriber" bash -c 'echo "Starting zmq-subscriber" ; ${ENV_PY} ${DIR}/zmq_subscriber.py; read x' echo -e $GREEN"\t* Launching zmq dispatcher"$DEFAULT -screen -S "$screenName" -X screen -t "zmq-dispatcher" bash -c 'echo "Starting zmq-dispatcher"; ./zmq_dispatcher.py; read x' +screen -S "$screenName" -X screen -t "zmq-dispatcher" bash -c 'echo "Starting zmq-dispatcher"; ${ENV_PY} ${DIR}/zmq_dispatcher.py; read x' echo -e $GREEN"\t* Launching flask server"$DEFAULT -screen -S "$screenName" -X screen -t "flask" bash -c 'echo "Starting Flask Server"; flask run --host=${FLASK_HOST} --port=${FLASK_PORT}; read x' +screen -S "$screenName" -X screen -t "flask" bash -c 'echo "Starting Flask Server"; ${ENV_PY} ${DIR}/server.py; read x' diff --git a/tests/start_framework.sh b/tests/start_framework.sh index 25ce738..82822e9 100755 --- a/tests/start_framework.sh +++ b/tests/start_framework.sh @@ -6,7 +6,8 @@ GREEN="\\033[1;32m" DEFAULT="\\033[0;39m" RED="\\033[1;31m" -[ -z "$DASH_HOME" ] && echo "Needs the env var DASHENV. Run the script from the virtual environment." && exit 1; +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +DASH_HOME="${DIR}/.." conf_dir="${DASH_HOME}/config/" redis_dir="${DASH_HOME}/../redis/src/" diff --git a/tests/terminate_framework.sh b/tests/terminate_framework.sh index f9cf426..31bfdf3 100755 --- a/tests/terminate_framework.sh +++ b/tests/terminate_framework.sh @@ -6,7 +6,9 @@ GREEN="\\033[1;32m" DEFAULT="\\033[0;39m" RED="\\033[1;31m" -[ -z "$DASH_HOME" ] && echo "Needs the env var DASHENV. Run the script from the virtual environment." && exit 1; +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +DASH_HOME="${DIR}/.. + conf_dir="${DASH_HOME}/config/" redis_dir="${DASH_HOME}/../redis/src/" diff --git a/tests/testall.sh b/tests/testall.sh index 2779766..7269434 100755 --- a/tests/testall.sh +++ b/tests/testall.sh @@ -1,6 +1,8 @@ #!/bin/bash set -e -[ -z "$DASH_HOME" ] && echo "Needs the env var DASHENV. Run the script from the virtual environment." && exit 1; +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +DASH_HOME="${DIR}/.." + ./start_framework.sh # Wait a bit that redis terminate From e070b94b4ef250d7c9fdc7e3b7c8cd289428d4dc Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Mon, 2 Apr 2018 13:10:21 +0900 Subject: [PATCH 08/18] =?UTF-8?q?-=20Maybe=20a=20fix=20for=20a=20very=20we?= =?UTF-8?q?ird=20bash=20behaviour=E2=80=A6=20see:=20http://pastebin.lu/ovi?= =?UTF-8?q?likesos.bash?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- start_all.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/start_all.sh b/start_all.sh index 6473228..776f186 100755 --- a/start_all.sh +++ b/start_all.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e set -x @@ -9,6 +9,7 @@ RED="\\033[1;31m" # Getting CWD where bash script resides DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +DASH_HOME=${DIR} if [ -e "${DIR}/DASHENV/bin/python" ]; then echo "dashboard virtualenv seems to exist, good" @@ -24,7 +25,7 @@ if [ ! -e "${redis_dir}" ]; then redis_dir="" fi -check_redis_port=$(netstat -an |grep LISTEN |grep 6250 |grep -v tcp6) +check_redis_port="$(netstat -an |grep LISTEN |grep 6250 |grep -v tcp6)" # Configure accordingly, remember: 0.0.0.0 exposes to every active IP interface, play safe and bind it to something you trust and know export FLASK_APP=server.py From 721fadba6053d1a09b154611b4ed13cc8c8fa370 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Mon, 2 Apr 2018 13:32:32 +0900 Subject: [PATCH 09/18] - Go to misp-dashboard home directory - Add a more sane bash way of checking for netstat results --- start_all.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/start_all.sh b/start_all.sh index 776f186..f2231eb 100755 --- a/start_all.sh +++ b/start_all.sh @@ -11,6 +11,8 @@ RED="\\033[1;31m" DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" DASH_HOME=${DIR} +cd ${DASH_HOME} + if [ -e "${DIR}/DASHENV/bin/python" ]; then echo "dashboard virtualenv seems to exist, good" ENV_PY=${DIR}/DASHENV/bin/python @@ -25,7 +27,7 @@ if [ ! -e "${redis_dir}" ]; then redis_dir="" fi -check_redis_port="$(netstat -an |grep LISTEN |grep 6250 |grep -v tcp6)" +netstat -an |grep LISTEN |grep 6250 |grep -v tcp6 ; check_redis_port=$? # Configure accordingly, remember: 0.0.0.0 exposes to every active IP interface, play safe and bind it to something you trust and know export FLASK_APP=server.py @@ -39,7 +41,7 @@ screenName="Misp-Dashboard" screen -dmS "$screenName" sleep 0.1 -if [ -z "${check_redis_port}" ]; then +if [ "${check_redis_port}" == "1" ]; then echo -e $GREEN"\t* Launching Redis servers"$DEFAULT screen -S "$screenName" -X screen -t "redis-server" bash -c $redis_dir'redis-server '$conf_dir'6250.conf && echo "Started Redis" ; read x' else From 0426b07a589ef255801c5fb5fa0c5cdc968c7cbe Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Mon, 2 Apr 2018 13:37:57 +0900 Subject: [PATCH 10/18] - Remove set -e, which prevents execution as soon as a process returns an exit code != 0 --- start_all.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/start_all.sh b/start_all.sh index f2231eb..17d550e 100755 --- a/start_all.sh +++ b/start_all.sh @@ -1,6 +1,5 @@ #!/usr/bin/env bash -set -e set -x GREEN="\\033[1;32m" From 03f2da36eb074efa0c520458eeb888e983fab8eb Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Mon, 2 Apr 2018 14:13:46 +0900 Subject: [PATCH 11/18] - After some buggering about, I think this will now finally work - bash -c wants to have positional parmeters, anything else is wrong, man bash: If the -c option is present, then commands are read from string. If there are arguments after the string, they are assigned to the positional parameters, starting with $0. --- start_all.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/start_all.sh b/start_all.sh index 17d550e..d12cd3f 100755 --- a/start_all.sh +++ b/start_all.sh @@ -8,13 +8,13 @@ RED="\\033[1;31m" # Getting CWD where bash script resides DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -DASH_HOME=${DIR} +DASH_HOME="${DIR}" cd ${DASH_HOME} if [ -e "${DIR}/DASHENV/bin/python" ]; then echo "dashboard virtualenv seems to exist, good" - ENV_PY=${DIR}/DASHENV/bin/python + ENV_PY="${DIR}/DASHENV/bin/python" else echo "Please make sure you have a dashboard environment, au revoir" exit 1 @@ -42,16 +42,16 @@ screen -dmS "$screenName" sleep 0.1 if [ "${check_redis_port}" == "1" ]; then echo -e $GREEN"\t* Launching Redis servers"$DEFAULT - screen -S "$screenName" -X screen -t "redis-server" bash -c $redis_dir'redis-server '$conf_dir'6250.conf && echo "Started Redis" ; read x' + screen -S "$screenName" -X screen -t "redis-server" bash -c '${0}redis-server ${1}6250.conf && echo "Started Redis" ; read x' ${redis_dir} ${conf_dir} else echo -e $RED"\t* NOT starting Redis server, made a very unrealiable check on port 6250, and something seems to be there… please double check if this is good!"$DEFAULT fi echo -e $GREEN"\t* Launching zmq subscriber"$DEFAULT -screen -S "$screenName" -X screen -t "zmq-subscriber" bash -c 'echo "Starting zmq-subscriber" ; ${ENV_PY} ${DIR}/zmq_subscriber.py; read x' +screen -S "$screenName" -X screen -t "zmq-subscriber" bash -c 'echo "Starting zmq-subscriber" ; ${0} ${1}'/zmq_subscriber.py; read x' ${ENV_PY} ${DIR} echo -e $GREEN"\t* Launching zmq dispatcher"$DEFAULT -screen -S "$screenName" -X screen -t "zmq-dispatcher" bash -c 'echo "Starting zmq-dispatcher"; ${ENV_PY} ${DIR}/zmq_dispatcher.py; read x' +screen -S "$screenName" -X screen -t "zmq-dispatcher" bash -c 'echo "Starting zmq-dispatcher"; ${0} ${1}/zmq_dispatcher.py; read x' ${ENV_PY} ${DIR} echo -e $GREEN"\t* Launching flask server"$DEFAULT -screen -S "$screenName" -X screen -t "flask" bash -c 'echo "Starting Flask Server"; ${ENV_PY} ${DIR}/server.py; read x' +screen -S "$screenName" -X screen -t "flask" bash -c 'echo "Starting Flask Server"; ${0} ${1}/server.py; read x' ${ENV_PY} ${DIR} From 1f23524dbc5d92c21be9c2db1af432382b72bc75 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Mon, 2 Apr 2018 14:42:06 +0900 Subject: [PATCH 12/18] - Small typo and remove absolute pathing due to "cd" in the beginning (not ideal but now good enough) --- start_all.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/start_all.sh b/start_all.sh index d12cd3f..3a60d57 100755 --- a/start_all.sh +++ b/start_all.sh @@ -34,7 +34,7 @@ export FLASK_DEBUG=0 export FLASK_PORT=8001 export FLASK_HOST=127.0.0.1 -conf_dir="${DASH_HOME}/config/" +conf_dir="config/" screenName="Misp-Dashboard" @@ -48,10 +48,10 @@ else fi echo -e $GREEN"\t* Launching zmq subscriber"$DEFAULT -screen -S "$screenName" -X screen -t "zmq-subscriber" bash -c 'echo "Starting zmq-subscriber" ; ${0} ${1}'/zmq_subscriber.py; read x' ${ENV_PY} ${DIR} +screen -S "$screenName" -X screen -t "zmq-subscriber" bash -c 'echo "Starting zmq-subscriber" ; ${0} ./zmq_subscriber.py; read x' ${ENV_PY} echo -e $GREEN"\t* Launching zmq dispatcher"$DEFAULT -screen -S "$screenName" -X screen -t "zmq-dispatcher" bash -c 'echo "Starting zmq-dispatcher"; ${0} ${1}/zmq_dispatcher.py; read x' ${ENV_PY} ${DIR} +screen -S "$screenName" -X screen -t "zmq-dispatcher" bash -c 'echo "Starting zmq-dispatcher"; ${0} ./zmq_dispatcher.py; read x' ${ENV_PY} echo -e $GREEN"\t* Launching flask server"$DEFAULT -screen -S "$screenName" -X screen -t "flask" bash -c 'echo "Starting Flask Server"; ${0} ${1}/server.py; read x' ${ENV_PY} ${DIR} +screen -S "$screenName" -X screen -t "flask" bash -c 'echo "Starting Flask Server"; ${0} ./server.py; read x' ${ENV_PY} From dba2c4cfeda80104e2570ed3303050fb2564a5ab Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Mon, 2 Apr 2018 15:20:52 +0900 Subject: [PATCH 13/18] =?UTF-8?q?-=20Gave=20up=20on=20screen,=20it=20is=20?= =?UTF-8?q?just=20not=20doable=20with=20my=20current=20skill-set=E2=80=A6?= =?UTF-8?q?=20-=20removed=20obsolete=20redis-server=20from=20source=20dete?= =?UTF-8?q?ction=20(redis=20is=20at=20a=20usuable=20version=20in=20latest?= =?UTF-8?q?=20ubuntu)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- start_all.sh | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/start_all.sh b/start_all.sh index 3a60d57..c24cd8b 100755 --- a/start_all.sh +++ b/start_all.sh @@ -20,11 +20,7 @@ else exit 1 fi -redis_dir="${DASH_HOME}/../redis/src/" -if [ ! -e "${redis_dir}" ]; then - [ ! -f "`which redis-server`" ] && echo "Either ${DASH_HOME}/../redis/src/ does not exist or 'redis-server' is not installed/not on PATH. Please fix and run again." && exit 1 - redis_dir="" -fi +[ ! -f "`which redis-server`" ] && echo "'redis-server' is not installed/not on PATH. Please fix and run again." && exit 1 netstat -an |grep LISTEN |grep 6250 |grep -v tcp6 ; check_redis_port=$? @@ -36,22 +32,19 @@ export FLASK_HOST=127.0.0.1 conf_dir="config/" -screenName="Misp-Dashboard" - -screen -dmS "$screenName" sleep 0.1 if [ "${check_redis_port}" == "1" ]; then echo -e $GREEN"\t* Launching Redis servers"$DEFAULT - screen -S "$screenName" -X screen -t "redis-server" bash -c '${0}redis-server ${1}6250.conf && echo "Started Redis" ; read x' ${redis_dir} ${conf_dir} + redis-server ${conf_dir}6250.conf & else echo -e $RED"\t* NOT starting Redis server, made a very unrealiable check on port 6250, and something seems to be there… please double check if this is good!"$DEFAULT fi echo -e $GREEN"\t* Launching zmq subscriber"$DEFAULT -screen -S "$screenName" -X screen -t "zmq-subscriber" bash -c 'echo "Starting zmq-subscriber" ; ${0} ./zmq_subscriber.py; read x' ${ENV_PY} +${ENV_PY} ./zmq_subscriber.py & echo -e $GREEN"\t* Launching zmq dispatcher"$DEFAULT -screen -S "$screenName" -X screen -t "zmq-dispatcher" bash -c 'echo "Starting zmq-dispatcher"; ${0} ./zmq_dispatcher.py; read x' ${ENV_PY} +${ENV_PY} ./zmq_dispatcher.py & echo -e $GREEN"\t* Launching flask server"$DEFAULT -screen -S "$screenName" -X screen -t "flask" bash -c 'echo "Starting Flask Server"; ${0} ./server.py; read x' ${ENV_PY} +${ENV_PY} ./server.py & From 4f5fe8d9bdd7178f81c854c23087ba25588a4b90 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 5 Apr 2018 09:49:12 +0200 Subject: [PATCH 14/18] - some formatting updates --- README.md | 4 +--- doc/{DOC.md => internals.md} | 0 2 files changed, 1 insertion(+), 3 deletions(-) rename doc/{DOC.md => internals.md} (100%) diff --git a/README.md b/README.md index 06367dc..c0f88aa 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ # MISP-Dashboard An experimental dashboard showing live data and statistics from the ZMQ of one or more MISP instances. - # Installation - Launch ```./install_dependencies.sh``` from the MISP-Dashboard directory - Update the configuration file ```config.cfg``` so that it matches your system @@ -38,8 +37,7 @@ OSError: [Errno 26] Text file busy: '/home/steve/code/misp-dashboard/DASHENV/bin ``` # Starting the System - -/!\ You do not need to run it as root. Normal privileges are fine. +:warning: You do not need to run it as root. Normal privileges are fine. - Be sure to have a running redis server - e.g. ```redis-server --port 6250``` diff --git a/doc/DOC.md b/doc/internals.md similarity index 100% rename from doc/DOC.md rename to doc/internals.md From 8f50423f89dc91b21654829c8954f37d89c96d6f Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 5 Apr 2018 16:07:27 +0200 Subject: [PATCH 15/18] - Added Licensing notes to 3/4/5.svg --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index c0f88aa..54e4cb3 100644 --- a/README.md +++ b/README.md @@ -199,6 +199,9 @@ Images and logos are handmade for: Note that: - Part of ```MISPHonorableIcons/1.svg``` comes from [octicons.github.com](https://octicons.github.com/icon/git-pull-request/) (CC0 - No Rights Reserved) - Part of ```MISPHonorableIcons/2.svg``` comes from [Zeptozephyr](https://zeptozephyr.deviantart.com/art/Vectored-Portal-Icons-207347804) (CC0 - No Rights Reserved) +- Part of ```MISPHonorableIcons/3.svg``` comes from [octicons.github.com](https://octicons.github.com/icon/git-pull-request/) (CC0 - No Rights Reserved) +- Part of ```MISPHonorableIcons/4.svg``` comes from [Zeptozephyr](https://zeptozephyr.deviantart.com/art/Vectored-Portal-Icons-207347804) & [octicons.github.com](https://octicons.github.com/icon/git-pull-request/) (CC0 - No Rights Reserved) +- Part of ```MISPHonorableIcons/5.svg``` comes from [Zeptozephyr](https://zeptozephyr.deviantart.com/art/Vectored-Portal-Icons-207347804) & [octicons.github.com](https://octicons.github.com/icon/git-pull-request/) (CC0 - No Rights Reserved) ``` Copyright (C) 2017 CIRCL - Computer Incident Response Center Luxembourg (c/o smile, security made in Lëtzebuerg, Groupement d'Intérêt Economique) From c71834ef30fee8dc9791038dca14af64b9e5876f Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 5 Apr 2018 16:25:10 +0200 Subject: [PATCH 16/18] =?UTF-8?q?-=20Added=20cake=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- static/pics/MISPHonorableIcons/5.svg | 1743 +++++++++++++++++++++++++- 1 file changed, 1714 insertions(+), 29 deletions(-) diff --git a/static/pics/MISPHonorableIcons/5.svg b/static/pics/MISPHonorableIcons/5.svg index a45efde..691242d 100644 --- a/static/pics/MISPHonorableIcons/5.svg +++ b/static/pics/MISPHonorableIcons/5.svg @@ -7,6 +7,7 @@ xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:svg="http://www.w3.org/2000/svg" xmlns="http://www.w3.org/2000/svg" + xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" sodipodi:docname="5.svg" @@ -197,9 +198,9 @@ showgrid="false" inkscape:current-layer="layer1" inkscape:document-units="mm" - inkscape:cy="202.03051" - inkscape:cx="193.94929" - inkscape:zoom="1.7687098" + inkscape:cy="54.842236" + inkscape:cx="218.80356" + inkscape:zoom="6.5835309" inkscape:pageshadow="2" inkscape:pageopacity="0.0" borderopacity="1.0" @@ -214,7 +215,7 @@ image/svg+xml - + @@ -361,31 +362,6 @@ style="fill:#2fa1db;fill-opacity:1" /> - - - - - - - + + + + + + + + + From 90543b716a4a451cbab1d3a213e468b29c2eaa06 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Fri, 6 Apr 2018 07:27:38 +0200 Subject: [PATCH 17/18] - Added some checks if "stuff" is running -- :warning: This works but is not 100% reliable :warning: --- start_all.sh | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/start_all.sh b/start_all.sh index c24cd8b..04b82c7 100755 --- a/start_all.sh +++ b/start_all.sh @@ -23,6 +23,9 @@ fi [ ! -f "`which redis-server`" ] && echo "'redis-server' is not installed/not on PATH. Please fix and run again." && exit 1 netstat -an |grep LISTEN |grep 6250 |grep -v tcp6 ; check_redis_port=$? +netstat -an |grep LISTEN |grep 8001 |grep -v tcp6 ; check_dashboard_port=$? +ps auxw |grep zmq_subscriber.py |grep -v grep ; check_zmq_subscriber=$? +ps auxw |grep zmq_dispatcher.py |grep -v grep ; check_zmq_dispatcher=$? # Configure accordingly, remember: 0.0.0.0 exposes to every active IP interface, play safe and bind it to something you trust and know export FLASK_APP=server.py @@ -40,11 +43,26 @@ else echo -e $RED"\t* NOT starting Redis server, made a very unrealiable check on port 6250, and something seems to be there… please double check if this is good!"$DEFAULT fi -echo -e $GREEN"\t* Launching zmq subscriber"$DEFAULT -${ENV_PY} ./zmq_subscriber.py & +sleep 0.1 +if [ "${check_zmq_subscriber}" == "1" ]; then + echo -e $GREEN"\t* Launching zmq subscriber"$DEFAULT + ${ENV_PY} ./zmq_subscriber.py & +else + echo -e $RED"\t* NOT starting zmq subscriber, made a rather unrealiable ps -auxw | grep for zmq_subscriber.py, and something seems to be there… please double check if this is good!"$DEFAULT +fi -echo -e $GREEN"\t* Launching zmq dispatcher"$DEFAULT -${ENV_PY} ./zmq_dispatcher.py & +sleep 0.1 +if [ "${check_zmq_dispatcher}" == "1" ]; then + echo -e $GREEN"\t* Launching zmq dispatcher"$DEFAULT + ${ENV_PY} ./zmq_dispatcher.py & +else + echo -e $RED"\t* NOT starting zmq dispatcher, made a rather unrealiable ps -auxw | grep for zmq_dispatcher.py, and something seems to be there… please double check if this is good!"$DEFAULT +fi -echo -e $GREEN"\t* Launching flask server"$DEFAULT -${ENV_PY} ./server.py & +sleep 0.1 +if [ "${check_dashboard_port}" == "1" ]; then + echo -e $GREEN"\t* Launching flask server"$DEFAULT + ${ENV_PY} ./server.py & +else + echo -e $RED"\t* NOT starting flask server, made a very unrealiable check on port 8001, and something seems to be there… please double check if this is good!"$DEFAULT +fi From 834a1b18d8113895c05810b6b76cbd2f1d15cb22 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Fri, 6 Apr 2018 07:52:34 +0200 Subject: [PATCH 18/18] - Fixed maxMindDb path in config.cfg creation" --- install_dependencies.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/install_dependencies.sh b/install_dependencies.sh index e52300b..5fe3265 100755 --- a/install_dependencies.sh +++ b/install_dependencies.sh @@ -24,6 +24,9 @@ if [ -e "config/config.cfg" ]; then fi else cp -i config/config.cfg.default config/config.cfg + echo "Sanitizing MaxMindDB Path" + sed -i "s|pathMaxMindDB=./data/GeoLite2-City/GeoLite2-City.mmdb|pathMaxMindDB=$PWD/data/GeoLite2-City/GeoLite2-City.mmdb|" config/config.cfg + sed -i "s|path_countrycode_to_coord_JSON=./data/country_code_lat_long.json|path_countrycode_to_coord_JSON=$PWD/data/country_code_lat_long.json|" config/config.cfg fi ## Web stuff