Merge pull request #97 from SteveClement/tryCatch

chg: [various] Added various try/excepts and split the log files into 3

config/config.cfg.default

@@ -33,7 +33,9 @@ additional_help_text = ["Sightings multiplies earned points by 2", "Editing an a
 
 [Log]
 directory=logs
-filename=logs.log
+dispatcher_filename=zmq_dispatcher.log
+subscriber_filename=zmq_subscriber.log
+helpers_filename=helpers.log
 
 [RedisGlobal]
 host=localhost

helpers/contributor_helper.py

@@ -33,11 +33,16 @@ class Contributor_helper:
 
         #logger
         logDir = cfg.get('Log', 'directory')
-        logfilename = cfg.get('Log', 'filename')
+        logfilename = cfg.get('Log', 'helpers_filename')
         logPath = os.path.join(logDir, logfilename)
         if not os.path.exists(logDir):
             os.makedirs(logDir)
-        logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
+        try:
+            logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
+        except PermissionError as error:
+            print(error)
+            print("Please fix the above and try again.")
+            sys.exit(126)
         self.logger = logging.getLogger(__name__)
 
         #honorBadge
@@ -123,14 +128,14 @@ class Contributor_helper:
         if action in ['edit', None]:
             pass
             #return #not a contribution?
-    
+
         now = datetime.datetime.now()
         nowSec = int(time.time())
         pnts_to_add = self.default_pnts_per_contribution
-    
+
         # Do not consider contribution as login anymore
         #self.users_helper.add_user_login(nowSec, org)
-    
+
         # is a valid contribution
         if categ is not None:
             try:
@@ -138,23 +143,23 @@ class Contributor_helper:
             except KeyError:
                 pnts_to_add = self.default_pnts_per_contribution
             pnts_to_add *= pntMultiplier
-    
+
             util.push_to_redis_zset(self.serv_redis_db, self.keyDay, org, count=pnts_to_add)
             #CONTRIB_CATEG retain the contribution per category, not the point earned in this categ
             util.push_to_redis_zset(self.serv_redis_db, self.keyCateg, org, count=1, endSubkey=':'+util.noSpaceLower(categ))
             self.publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'categ': categ, 'action': action, 'epoch': nowSec }, channel=self.CHANNEL_LASTCONTRIB)
         else:
             categ = ""
-    
+
         self.serv_redis_db.sadd(self.keyAllOrg, org)
-    
+
         keyname = "{}:{}".format(self.keyLastContrib, util.getDateStrFormat(now))
         self.serv_redis_db.zadd(keyname, nowSec, org)
         self.logger.debug('Added to redis: keyname={}, nowSec={}, org={}'.format(keyname, nowSec, org))
         self.serv_redis_db.expire(keyname, util.ONE_DAY*7) #expire after 7 day
-    
+
         awards_given = self.updateOrgContributionRank(org, pnts_to_add, action, contribType, eventTime=datetime.datetime.now(), isLabeled=isLabeled, categ=util.noSpaceLower(categ))
-    
+
         for award in awards_given:
             # update awards given
             keyname = "{}:{}".format(self.keyLastAward, util.getDateStrFormat(now))

helpers/geo_helper.py

@@ -32,11 +32,16 @@ class Geo_helper:
 
         #logger
         logDir = cfg.get('Log', 'directory')
-        logfilename = cfg.get('Log', 'filename')
+        logfilename = cfg.get('Log', 'helpers_filename')
         logPath = os.path.join(logDir, logfilename)
         if not os.path.exists(logDir):
             os.makedirs(logDir)
-        logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
+        try:
+            logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
+        except PermissionError as error:
+            print(error)
+            print("Please fix the above and try again.")
+            sys.exit(126)
         self.logger = logging.getLogger(__name__)
 
         self.keyCategCoord = "GEO_COORD"
@@ -46,7 +51,12 @@ class Geo_helper:
         self.PATH_TO_JSON = cfg.get('RedisMap', 'path_countrycode_to_coord_JSON')
         self.CHANNELDISP = cfg.get('RedisMap', 'channelDisp')
 
-        self.reader = geoip2.database.Reader(self.PATH_TO_DB)
+        try:
+            self.reader = geoip2.database.Reader(self.PATH_TO_DB)
+        except PermissionError as error:
+            print(error)
+            print("Please fix the above and try again.")
+            sys.exit(126)
         self.country_to_iso = { country.name: country.alpha_2 for country in pycountry.countries}
         with open(self.PATH_TO_JSON) as f:
             self.country_code_to_coord = json.load(f)
@@ -128,7 +138,7 @@ class Geo_helper:
             self.live_helper.add_to_stream_log_cache('Map', j_to_send)
             self.logger.info('Published: {}'.format(json.dumps(to_send)))
         except ValueError:
-            self.logger.warning("can't resolve ip")
+            self.logger.warning("Can't resolve IP: " + str(supposed_ip))
         except geoip2.errors.AddressNotFoundError:
             self.logger.warning("Address not in Database")
         except InvalidCoordinate:
@@ -184,7 +194,12 @@ class Geo_helper:
         now = datetime.datetime.now()
         today_str = util.getDateStrFormat(now)
         keyname = "{}:{}".format(keyCateg, today_str)
-        self.serv_redis_db.geoadd(keyname, lon, lat, content)
+        try:
+            self.serv_redis_db.geoadd(keyname, lon, lat, content)
+        except redis.exceptions.ResponseError as error:
+            print(error)
+            print("Please fix the above, and make sure you use a redis version that supports the GEOADD command.")
+            print("To test for support: echo \"help GEOADD\"| redis-cli")
         self.logger.debug('Added to redis: keyname={}, lon={}, lat={}, content={}'.format(keyname, lon, lat, content))
     def push_to_redis_zset(self, keyCateg, toAdd, endSubkey="", count=1):
         now = datetime.datetime.now()

helpers/live_helper.py

@@ -17,11 +17,16 @@ class Live_helper:
 
         # logger
         logDir = cfg.get('Log', 'directory')
-        logfilename = cfg.get('Log', 'filename')
+        logfilename = cfg.get('Log', 'helpers_filename')
         logPath = os.path.join(logDir, logfilename)
         if not os.path.exists(logDir):
             os.makedirs(logDir)
-        logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
+        try:
+            logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
+        except PermissionError as error:
+            print(error)
+            print("Please fix the above and try again.")
+            sys.exit(126)
         self.logger = logging.getLogger(__name__)
 
     def publish_log(self, zmq_name, name, content, channel=None):
@@ -44,7 +49,7 @@ class Live_helper:
             jentry = json.loads(entry.decode('utf8'))
             to_ret.append(jentry)
         return to_ret
-    
+
 
     def add_to_stream_log_cache(self, cacheKey, item):
         rKey = self.prefix_redis_key+cacheKey

helpers/trendings_helper.py

@@ -26,11 +26,16 @@ class Trendings_helper:
 
         #logger
         logDir = cfg.get('Log', 'directory')
-        logfilename = cfg.get('Log', 'filename')
+        logfilename = cfg.get('Log', 'helpers_filename')
         logPath = os.path.join(logDir, logfilename)
         if not os.path.exists(logDir):
             os.makedirs(logDir)
-        logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
+        try:
+            logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
+        except PermissionError as error:
+            print(error)
+            print("Please fix the above and try again.")
+            sys.exit(126)
         self.logger = logging.getLogger(__name__)
 
     ''' SETTER '''

helpers/users_helper.py

@@ -23,11 +23,16 @@ class Users_helper:
 
         #logger
         logDir = cfg.get('Log', 'directory')
-        logfilename = cfg.get('Log', 'filename')
+        logfilename = cfg.get('Log', 'helpers_filename')
         logPath = os.path.join(logDir, logfilename)
         if not os.path.exists(logDir):
             os.makedirs(logDir)
-        logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
+        try:
+            logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
+        except PermissionError as error:
+            print(error)
+            print("Please fix the above and try again.")
+            sys.exit(126)
         self.logger = logging.getLogger(__name__)
 
     def add_user_login(self, timestamp, org, email=''):

install_dependencies.sh

@@ -1,12 +1,21 @@
 #!/bin/bash
 
-set -e
+## disable -e for production systems
+#set -e
+
+## Debug mode
 #set -x
 
 sudo apt-get install python3-virtualenv virtualenv screen redis-server unzip -y
 
 if [ -z "$VIRTUAL_ENV" ]; then
-    virtualenv -p python3 DASHENV
+    virtualenv -p python3 DASHENV ; DASH_VENV=$?
+
+    if [[ "$DASH_VENV" != "0" ]]; then
+      echo "Something went wrong with either the update or install of the virtualenv."
+      echo "Please investigate manually."
+      exit $DASH_VENV
+    fi
 
     . ./DASHENV/bin/activate
 fi
@@ -35,7 +44,14 @@ mkdir -p css fonts js
 popd
 mkdir -p temp
 
-wget http://www.misp-project.org/assets/images/misp-small.png -O static/pics/MISP.png
+NET_WGET=$(wget --no-cache -q https://www.misp-project.org/assets/images/misp-small.png -O static/pics/MISP.png; echo $?)
+
+if [[ "$NET_WGET" != "0" ]]; then
+  echo "The first wget we tried failed, please investigate manually."
+  exit $NET_WGET
+fi
+
+wget https://www.misp-project.org/favicon.ico -O static/favicon.ico
 
 # jquery
 JQVERSION="3.2.1"

server.py

@@ -14,7 +14,7 @@ import redis
 
 import util
 from flask import (Flask, Response, jsonify, render_template, request,
-                   stream_with_context)
+                   send_from_directory, stream_with_context)
 from helpers import (contributor_helper, geo_helper, live_helper,
                      trendings_helper, users_helper)
 
@@ -141,6 +141,10 @@ def index():
             zoomlevel=cfg.getint('Dashboard' ,'zoomlevel')
             )
 
+@app.route('/favicon.ico')
+def favicon():
+    return send_from_directory(os.path.join(app.root_path, 'static'),
+                               'favicon.ico', mimetype='image/vnd.microsoft.icon')
 
 @app.route("/geo")
 def geo():

zmq_dispatcher.py

@@ -23,11 +23,16 @@ cfg = configparser.ConfigParser()
 cfg.read(configfile)
 
 logDir = cfg.get('Log', 'directory')
-logfilename = cfg.get('Log', 'filename')
+logfilename = cfg.get('Log', 'dispatcher_filename')
 logPath = os.path.join(logDir, logfilename)
 if not os.path.exists(logDir):
     os.makedirs(logDir)
-logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
+try:
+    logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
+except PermissionError as error:
+    print(error)
+    print("Please fix the above and try again.")
+    sys.exit(126)
 logger = logging.getLogger('zmq_dispatcher')
 
 LISTNAME = cfg.get('RedisLIST', 'listName')
@@ -289,4 +294,7 @@ if __name__ == "__main__":
     parser.add_argument('-s', '--sleep', required=False, dest='sleeptime', type=int, help='The number of second to wait before checking redis list size', default=5)
     args = parser.parse_args()
 
-    main(args.sleeptime)
+    try:
+        main(args.sleeptime)
+    except (redis.exceptions.ResponseError, KeyboardInterrupt) as error:
+        print(error)

zmq_subscriber.py

@@ -16,11 +16,16 @@ configfile = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'config/c
 cfg = configparser.ConfigParser()
 cfg.read(configfile)
 logDir = cfg.get('Log', 'directory')
-logfilename = cfg.get('Log', 'filename')
+logfilename = cfg.get('Log', 'subscriber_filename')
 logPath = os.path.join(logDir, logfilename)
 if not os.path.exists(logDir):
     os.makedirs(logDir)
-logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
+try:
+    logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
+except PermissionError as error:
+    print(error)
+    print("Please fix the above and try again.")
+    sys.exit(126)
 logger = logging.getLogger('zmq_subscriber')
 
 ZMQ_URL = cfg.get('RedisGlobal', 'zmq_url')
@@ -59,9 +64,12 @@ def main(zmqName):
 
 if __name__ == "__main__":
 
-    parser = argparse.ArgumentParser(description='A zmq subscriber. It subscribes to a ZNQ then redispatch it to the misp-dashboard')
+    parser = argparse.ArgumentParser(description='A zmq subscriber. It subscribes to a ZMQ then redispatch it to the misp-dashboard')
     parser.add_argument('-n', '--name', required=False, dest='zmqname', help='The ZMQ feed name', default="MISP Standard ZMQ")
     parser.add_argument('-u', '--url', required=False, dest='zmqurl', help='The URL to connect to', default=ZMQ_URL)
     args = parser.parse_args()
 
-    main(args.zmqname)
+    try:
+        main(args.zmqname)
+    except redis.exceptions.ResponseError as error:
+        print(error)