From fc1c83242065176105c7f03b1bf9ae89c4df636e Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Wed, 29 May 2019 09:48:26 +0900 Subject: [PATCH 1/6] fix: [errorHandling] more try catch error --- zmq_dispatcher.py | 12 ++++++++++-- zmq_subscriber.py | 12 ++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/zmq_dispatcher.py b/zmq_dispatcher.py index 3f94b81..d5892f3 100755 --- a/zmq_dispatcher.py +++ b/zmq_dispatcher.py @@ -27,7 +27,12 @@ logfilename = cfg.get('Log', 'filename') logPath = os.path.join(logDir, logfilename) if not os.path.exists(logDir): os.makedirs(logDir) -logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO) +try: + logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO) +except PermissionError as error: + print(error) + print("Please fix the above and try again.") + sys.exit(126) logger = logging.getLogger('zmq_dispatcher') LISTNAME = cfg.get('RedisLIST', 'listName') @@ -289,4 +294,7 @@ if __name__ == "__main__": parser.add_argument('-s', '--sleep', required=False, dest='sleeptime', type=int, help='The number of second to wait before checking redis list size', default=5) args = parser.parse_args() - main(args.sleeptime) + try: + main(args.sleeptime) + except redis.exceptions.ResponseError as error: + print(error) diff --git a/zmq_subscriber.py b/zmq_subscriber.py index 97b2a71..f793040 100755 --- a/zmq_subscriber.py +++ b/zmq_subscriber.py @@ -20,7 +20,12 @@ logfilename = cfg.get('Log', 'filename') logPath = os.path.join(logDir, logfilename) if not os.path.exists(logDir): os.makedirs(logDir) -logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO) +try: + logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO) +except PermissionError as error: + print(error) + print("Please fix the above and try again.") + sys.exit(126) logger = logging.getLogger('zmq_subscriber') ZMQ_URL = cfg.get('RedisGlobal', 'zmq_url') @@ -64,4 +69,7 @@ if __name__ == "__main__": parser.add_argument('-u', '--url', required=False, dest='zmqurl', help='The URL to connect to', default=ZMQ_URL) args = parser.parse_args() - main(args.zmqname) + try: + main(args.zmqname) + except redis.exceptions.ResponseError as error: + print(error) From 6b291a55263a46ba599eddb48b36a480bd4aa34e Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Wed, 29 May 2019 09:58:09 +0900 Subject: [PATCH 2/6] fix: [geoadd] Catch the following issue: https://github.com/MISP/misp-dashboard/issues/70 --- helpers/geo_helper.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/helpers/geo_helper.py b/helpers/geo_helper.py index 57a0065..2f59066 100644 --- a/helpers/geo_helper.py +++ b/helpers/geo_helper.py @@ -184,7 +184,12 @@ class Geo_helper: now = datetime.datetime.now() today_str = util.getDateStrFormat(now) keyname = "{}:{}".format(keyCateg, today_str) - self.serv_redis_db.geoadd(keyname, lon, lat, content) + try: + self.serv_redis_db.geoadd(keyname, lon, lat, content) + except redis.exceptions.ResponseError as error: + print(error) + print("Please fix the above, and make sure you use a redis version that supports the GEOADD command.") + print("To test for support: echo \"help GEOADD\"| redis-cli") self.logger.debug('Added to redis: keyname={}, lon={}, lat={}, content={}'.format(keyname, lon, lat, content)) def push_to_redis_zset(self, keyCateg, toAdd, endSubkey="", count=1): now = datetime.datetime.now() From 92b91c5c05dcb373487b32e73241d1741988d60f Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Wed, 29 May 2019 10:41:43 +0900 Subject: [PATCH 3/6] chg: [log] Added 3 seperate log files: helpers.log / zmq_subscribre.log / zmq_dispatcher.log fix: [log] Catch permission errors on log files --- config/config.cfg.default | 4 +++- helpers/contributor_helper.py | 25 +++++++++++++++---------- helpers/geo_helper.py | 9 +++++++-- helpers/live_helper.py | 11 ++++++++--- helpers/trendings_helper.py | 9 +++++++-- helpers/users_helper.py | 9 +++++++-- zmq_dispatcher.py | 4 ++-- zmq_subscriber.py | 4 ++-- 8 files changed, 51 insertions(+), 24 deletions(-) diff --git a/config/config.cfg.default b/config/config.cfg.default index 1d18adb..7597d26 100644 --- a/config/config.cfg.default +++ b/config/config.cfg.default @@ -33,7 +33,9 @@ additional_help_text = ["Sightings multiplies earned points by 2", "Editing an a [Log] directory=logs -filename=logs.log +dispatcher_filename=zmq_dispatcher.log +subscriber_filename=zmq_subscriber.log +helpers_filename=helpers.log [RedisGlobal] host=localhost diff --git a/helpers/contributor_helper.py b/helpers/contributor_helper.py index 5dd6abc..68bf16a 100644 --- a/helpers/contributor_helper.py +++ b/helpers/contributor_helper.py @@ -33,11 +33,16 @@ class Contributor_helper: #logger logDir = cfg.get('Log', 'directory') - logfilename = cfg.get('Log', 'filename') + logfilename = cfg.get('Log', 'helpers_filename') logPath = os.path.join(logDir, logfilename) if not os.path.exists(logDir): os.makedirs(logDir) - logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO) + try: + logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO) + except PermissionError as error: + print(error) + print("Please fix the above and try again.") + sys.exit(126) self.logger = logging.getLogger(__name__) #honorBadge @@ -123,14 +128,14 @@ class Contributor_helper: if action in ['edit', None]: pass #return #not a contribution? - + now = datetime.datetime.now() nowSec = int(time.time()) pnts_to_add = self.default_pnts_per_contribution - + # Do not consider contribution as login anymore #self.users_helper.add_user_login(nowSec, org) - + # is a valid contribution if categ is not None: try: @@ -138,23 +143,23 @@ class Contributor_helper: except KeyError: pnts_to_add = self.default_pnts_per_contribution pnts_to_add *= pntMultiplier - + util.push_to_redis_zset(self.serv_redis_db, self.keyDay, org, count=pnts_to_add) #CONTRIB_CATEG retain the contribution per category, not the point earned in this categ util.push_to_redis_zset(self.serv_redis_db, self.keyCateg, org, count=1, endSubkey=':'+util.noSpaceLower(categ)) self.publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'categ': categ, 'action': action, 'epoch': nowSec }, channel=self.CHANNEL_LASTCONTRIB) else: categ = "" - + self.serv_redis_db.sadd(self.keyAllOrg, org) - + keyname = "{}:{}".format(self.keyLastContrib, util.getDateStrFormat(now)) self.serv_redis_db.zadd(keyname, nowSec, org) self.logger.debug('Added to redis: keyname={}, nowSec={}, org={}'.format(keyname, nowSec, org)) self.serv_redis_db.expire(keyname, util.ONE_DAY*7) #expire after 7 day - + awards_given = self.updateOrgContributionRank(org, pnts_to_add, action, contribType, eventTime=datetime.datetime.now(), isLabeled=isLabeled, categ=util.noSpaceLower(categ)) - + for award in awards_given: # update awards given keyname = "{}:{}".format(self.keyLastAward, util.getDateStrFormat(now)) diff --git a/helpers/geo_helper.py b/helpers/geo_helper.py index 2f59066..b2f5de3 100644 --- a/helpers/geo_helper.py +++ b/helpers/geo_helper.py @@ -32,11 +32,16 @@ class Geo_helper: #logger logDir = cfg.get('Log', 'directory') - logfilename = cfg.get('Log', 'filename') + logfilename = cfg.get('Log', 'helpers_filename') logPath = os.path.join(logDir, logfilename) if not os.path.exists(logDir): os.makedirs(logDir) - logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO) + try: + logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO) + except PermissionError as error: + print(error) + print("Please fix the above and try again.") + sys.exit(126) self.logger = logging.getLogger(__name__) self.keyCategCoord = "GEO_COORD" diff --git a/helpers/live_helper.py b/helpers/live_helper.py index 31af76b..cda4eda 100644 --- a/helpers/live_helper.py +++ b/helpers/live_helper.py @@ -17,11 +17,16 @@ class Live_helper: # logger logDir = cfg.get('Log', 'directory') - logfilename = cfg.get('Log', 'filename') + logfilename = cfg.get('Log', 'helpers_filename') logPath = os.path.join(logDir, logfilename) if not os.path.exists(logDir): os.makedirs(logDir) - logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO) + try: + logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO) + except PermissionError as error: + print(error) + print("Please fix the above and try again.") + sys.exit(126) self.logger = logging.getLogger(__name__) def publish_log(self, zmq_name, name, content, channel=None): @@ -44,7 +49,7 @@ class Live_helper: jentry = json.loads(entry.decode('utf8')) to_ret.append(jentry) return to_ret - + def add_to_stream_log_cache(self, cacheKey, item): rKey = self.prefix_redis_key+cacheKey diff --git a/helpers/trendings_helper.py b/helpers/trendings_helper.py index e19292c..07e91e2 100644 --- a/helpers/trendings_helper.py +++ b/helpers/trendings_helper.py @@ -26,11 +26,16 @@ class Trendings_helper: #logger logDir = cfg.get('Log', 'directory') - logfilename = cfg.get('Log', 'filename') + logfilename = cfg.get('Log', 'helpers_filename') logPath = os.path.join(logDir, logfilename) if not os.path.exists(logDir): os.makedirs(logDir) - logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO) + try: + logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO) + except PermissionError as error: + print(error) + print("Please fix the above and try again.") + sys.exit(126) self.logger = logging.getLogger(__name__) ''' SETTER ''' diff --git a/helpers/users_helper.py b/helpers/users_helper.py index f38b1f9..18498bc 100644 --- a/helpers/users_helper.py +++ b/helpers/users_helper.py @@ -23,11 +23,16 @@ class Users_helper: #logger logDir = cfg.get('Log', 'directory') - logfilename = cfg.get('Log', 'filename') + logfilename = cfg.get('Log', 'helpers_filename') logPath = os.path.join(logDir, logfilename) if not os.path.exists(logDir): os.makedirs(logDir) - logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO) + try: + logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO) + except PermissionError as error: + print(error) + print("Please fix the above and try again.") + sys.exit(126) self.logger = logging.getLogger(__name__) def add_user_login(self, timestamp, org, email=''): diff --git a/zmq_dispatcher.py b/zmq_dispatcher.py index d5892f3..7c9b5d8 100755 --- a/zmq_dispatcher.py +++ b/zmq_dispatcher.py @@ -23,7 +23,7 @@ cfg = configparser.ConfigParser() cfg.read(configfile) logDir = cfg.get('Log', 'directory') -logfilename = cfg.get('Log', 'filename') +logfilename = cfg.get('Log', 'dispatcher_filename') logPath = os.path.join(logDir, logfilename) if not os.path.exists(logDir): os.makedirs(logDir) @@ -296,5 +296,5 @@ if __name__ == "__main__": try: main(args.sleeptime) - except redis.exceptions.ResponseError as error: + except (redis.exceptions.ResponseError, KeyboardInterrupt) as error: print(error) diff --git a/zmq_subscriber.py b/zmq_subscriber.py index f793040..ad9e548 100755 --- a/zmq_subscriber.py +++ b/zmq_subscriber.py @@ -16,7 +16,7 @@ configfile = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'config/c cfg = configparser.ConfigParser() cfg.read(configfile) logDir = cfg.get('Log', 'directory') -logfilename = cfg.get('Log', 'filename') +logfilename = cfg.get('Log', 'subscriber_filename') logPath = os.path.join(logDir, logfilename) if not os.path.exists(logDir): os.makedirs(logDir) @@ -64,7 +64,7 @@ def main(zmqName): if __name__ == "__main__": - parser = argparse.ArgumentParser(description='A zmq subscriber. It subscribes to a ZNQ then redispatch it to the misp-dashboard') + parser = argparse.ArgumentParser(description='A zmq subscriber. It subscribes to a ZMQ then redispatch it to the misp-dashboard') parser.add_argument('-n', '--name', required=False, dest='zmqname', help='The ZMQ feed name', default="MISP Standard ZMQ") parser.add_argument('-u', '--url', required=False, dest='zmqurl', help='The URL to connect to', default=ZMQ_URL) args = parser.parse_args() From a65d2cb233488706929e6d0e37233045e7ec8765 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Wed, 29 May 2019 10:58:09 +0900 Subject: [PATCH 4/6] chg: [log] Let the user know which "IP" was not resolved. --- helpers/geo_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/geo_helper.py b/helpers/geo_helper.py index b2f5de3..d858111 100644 --- a/helpers/geo_helper.py +++ b/helpers/geo_helper.py @@ -133,7 +133,7 @@ class Geo_helper: self.live_helper.add_to_stream_log_cache('Map', j_to_send) self.logger.info('Published: {}'.format(json.dumps(to_send))) except ValueError: - self.logger.warning("can't resolve ip") + self.logger.warning("Can't resolve IP: " + str(supposed_ip)) except geoip2.errors.AddressNotFoundError: self.logger.warning("Address not in Database") except InvalidCoordinate: From cba4ab68f1cd6732ccff1202303f550a3a7a2ea0 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Wed, 29 May 2019 11:18:32 +0900 Subject: [PATCH 5/6] fix: [flask] Added favicon.ico --- install_dependencies.sh | 3 ++- server.py | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/install_dependencies.sh b/install_dependencies.sh index e9beb8e..51d86b6 100755 --- a/install_dependencies.sh +++ b/install_dependencies.sh @@ -35,7 +35,8 @@ mkdir -p css fonts js popd mkdir -p temp -wget http://www.misp-project.org/assets/images/misp-small.png -O static/pics/MISP.png +wget https://www.misp-project.org/assets/images/misp-small.png -O static/pics/MISP.png +wget https://www.misp-project.org/favicon.ico -O static/favicon.ico # jquery JQVERSION="3.2.1" diff --git a/server.py b/server.py index e7da6a6..ab2a2e5 100755 --- a/server.py +++ b/server.py @@ -14,7 +14,7 @@ import redis import util from flask import (Flask, Response, jsonify, render_template, request, - stream_with_context) + send_from_directory, stream_with_context) from helpers import (contributor_helper, geo_helper, live_helper, trendings_helper, users_helper) @@ -141,6 +141,10 @@ def index(): zoomlevel=cfg.getint('Dashboard' ,'zoomlevel') ) +@app.route('/favicon.ico') +def favicon(): + return send_from_directory(os.path.join(app.root_path, 'static'), + 'favicon.ico', mimetype='image/vnd.microsoft.icon') @app.route("/geo") def geo(): From 1f48d9621cf948184b66978afbe9dcead139b40d Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Wed, 29 May 2019 13:15:49 +0900 Subject: [PATCH 6/6] chg: [perms] Check if permissions fail on the MaxMind db files chg: [installer] Check if network is present and the first wget worked chg: [installer] Exit if the virtualenv creation/update fails --- helpers/geo_helper.py | 7 ++++++- install_dependencies.sh | 21 ++++++++++++++++++--- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/helpers/geo_helper.py b/helpers/geo_helper.py index d858111..8e8af52 100644 --- a/helpers/geo_helper.py +++ b/helpers/geo_helper.py @@ -51,7 +51,12 @@ class Geo_helper: self.PATH_TO_JSON = cfg.get('RedisMap', 'path_countrycode_to_coord_JSON') self.CHANNELDISP = cfg.get('RedisMap', 'channelDisp') - self.reader = geoip2.database.Reader(self.PATH_TO_DB) + try: + self.reader = geoip2.database.Reader(self.PATH_TO_DB) + except PermissionError as error: + print(error) + print("Please fix the above and try again.") + sys.exit(126) self.country_to_iso = { country.name: country.alpha_2 for country in pycountry.countries} with open(self.PATH_TO_JSON) as f: self.country_code_to_coord = json.load(f) diff --git a/install_dependencies.sh b/install_dependencies.sh index 51d86b6..3a0747d 100755 --- a/install_dependencies.sh +++ b/install_dependencies.sh @@ -1,12 +1,21 @@ #!/bin/bash -set -e +## disable -e for production systems +#set -e + +## Debug mode #set -x sudo apt-get install python3-virtualenv virtualenv screen redis-server unzip -y if [ -z "$VIRTUAL_ENV" ]; then - virtualenv -p python3 DASHENV + virtualenv -p python3 DASHENV ; DASH_VENV=$? + + if [[ "$DASH_VENV" != "0" ]]; then + echo "Something went wrong with either the update or install of the virtualenv." + echo "Please investigate manually." + exit $DASH_VENV + fi . ./DASHENV/bin/activate fi @@ -35,7 +44,13 @@ mkdir -p css fonts js popd mkdir -p temp -wget https://www.misp-project.org/assets/images/misp-small.png -O static/pics/MISP.png +NET_WGET=$(wget --no-cache -q https://www.misp-project.org/assets/images/misp-small.png -O static/pics/MISP.png; echo $?) + +if [[ "$NET_WGET" != "0" ]]; then + echo "The first wget we tried failed, please investigate manually." + exit $NET_WGET +fi + wget https://www.misp-project.org/favicon.ico -O static/favicon.ico # jquery