MISP
/
misp-dashboard
mirror of https://github.com/MISP/misp-dashboard
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #97 from SteveClement/tryCatch 

chg: [various] Added various try/excepts and split the log files into 3
pull/101/head
Steve Clement 2019-05-30 02:34:04 +02:00 committed by GitHub
parent b0a4f5d17b f22d3075ec
commit e9b1339967
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 107 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
config/config.cfg.default
View File

@ -33,7 +33,9 @@ additional_help_text = ["Sightings multiplies earned points by 2", "Editing an a
[Log]
directory=logs
filename=logs.log
dispatcher_filename=zmq_dispatcher.log
subscriber_filename=zmq_subscriber.log
helpers_filename=helpers.log
[RedisGlobal]
host=localhost

25
helpers/contributor_helper.py
View File

@ -33,11 +33,16 @@ class Contributor_helper:
#logger
logDir = cfg.get('Log', 'directory')
logfilename = cfg.get('Log', 'filename')
logfilename = cfg.get('Log', 'helpers_filename')
logPath = os.path.join(logDir, logfilename)
if not os.path.exists(logDir):
os.makedirs(logDir)
logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
try:
logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
except PermissionError as error:
print(error)
print("Please fix the above and try again.")
sys.exit(126)
self.logger = logging.getLogger(__name__)
#honorBadge
@ -123,14 +128,14 @@ class Contributor_helper:
if action in ['edit', None]:
pass
#return #not a contribution?
now = datetime.datetime.now()
nowSec = int(time.time())
pnts_to_add = self.default_pnts_per_contribution
# Do not consider contribution as login anymore
#self.users_helper.add_user_login(nowSec, org)
# is a valid contribution
if categ is not None:
try:
@ -138,23 +143,23 @@ class Contributor_helper:
except KeyError:
pnts_to_add = self.default_pnts_per_contribution
pnts_to_add *= pntMultiplier
util.push_to_redis_zset(self.serv_redis_db, self.keyDay, org, count=pnts_to_add)
#CONTRIB_CATEG retain the contribution per category, not the point earned in this categ
util.push_to_redis_zset(self.serv_redis_db, self.keyCateg, org, count=1, endSubkey=':'+util.noSpaceLower(categ))
self.publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'categ': categ, 'action': action, 'epoch': nowSec }, channel=self.CHANNEL_LASTCONTRIB)
else:
categ = ""
self.serv_redis_db.sadd(self.keyAllOrg, org)
keyname = "{}:{}".format(self.keyLastContrib, util.getDateStrFormat(now))
self.serv_redis_db.zadd(keyname, nowSec, org)
self.logger.debug('Added to redis: keyname={}, nowSec={}, org={}'.format(keyname, nowSec, org))
self.serv_redis_db.expire(keyname, util.ONE_DAY*7) #expire after 7 day
awards_given = self.updateOrgContributionRank(org, pnts_to_add, action, contribType, eventTime=datetime.datetime.now(), isLabeled=isLabeled, categ=util.noSpaceLower(categ))
for award in awards_given:
# update awards given
keyname = "{}:{}".format(self.keyLastAward, util.getDateStrFormat(now))

25
helpers/geo_helper.py
View File

@ -32,11 +32,16 @@ class Geo_helper:
#logger
logDir = cfg.get('Log', 'directory')
logfilename = cfg.get('Log', 'filename')
logfilename = cfg.get('Log', 'helpers_filename')
logPath = os.path.join(logDir, logfilename)
if not os.path.exists(logDir):
os.makedirs(logDir)
logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
try:
logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
except PermissionError as error:
print(error)
print("Please fix the above and try again.")
sys.exit(126)
self.logger = logging.getLogger(__name__)
self.keyCategCoord = "GEO_COORD"
@ -46,7 +51,12 @@ class Geo_helper:
self.PATH_TO_JSON = cfg.get('RedisMap', 'path_countrycode_to_coord_JSON')
self.CHANNELDISP = cfg.get('RedisMap', 'channelDisp')
self.reader = geoip2.database.Reader(self.PATH_TO_DB)
try:
self.reader = geoip2.database.Reader(self.PATH_TO_DB)
except PermissionError as error:
print(error)
print("Please fix the above and try again.")
sys.exit(126)
self.country_to_iso = { country.name: country.alpha_2 for country in pycountry.countries}
with open(self.PATH_TO_JSON) as f:
self.country_code_to_coord = json.load(f)
@ -128,7 +138,7 @@ class Geo_helper:
self.live_helper.add_to_stream_log_cache('Map', j_to_send)
self.logger.info('Published: {}'.format(json.dumps(to_send)))
except ValueError:
self.logger.warning("can't resolve ip")
self.logger.warning("Can't resolve IP: " + str(supposed_ip))
except geoip2.errors.AddressNotFoundError:
self.logger.warning("Address not in Database")
except InvalidCoordinate:
@ -184,7 +194,12 @@ class Geo_helper:
now = datetime.datetime.now()
today_str = util.getDateStrFormat(now)
keyname = "{}:{}".format(keyCateg, today_str)
self.serv_redis_db.geoadd(keyname, lon, lat, content)
try:
self.serv_redis_db.geoadd(keyname, lon, lat, content)
except redis.exceptions.ResponseError as error:
print(error)
print("Please fix the above, and make sure you use a redis version that supports the GEOADD command.")
print("To test for support: echo \"help GEOADD\"| redis-cli")
self.logger.debug('Added to redis: keyname={}, lon={}, lat={}, content={}'.format(keyname, lon, lat, content))
def push_to_redis_zset(self, keyCateg, toAdd, endSubkey="", count=1):
now = datetime.datetime.now()

11
helpers/live_helper.py
View File

@ -17,11 +17,16 @@ class Live_helper:
# logger
logDir = cfg.get('Log', 'directory')
logfilename = cfg.get('Log', 'filename')
logfilename = cfg.get('Log', 'helpers_filename')
logPath = os.path.join(logDir, logfilename)
if not os.path.exists(logDir):
os.makedirs(logDir)
logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
try:
logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
except PermissionError as error:
print(error)
print("Please fix the above and try again.")
sys.exit(126)
self.logger = logging.getLogger(__name__)
def publish_log(self, zmq_name, name, content, channel=None):
@ -44,7 +49,7 @@ class Live_helper:
jentry = json.loads(entry.decode('utf8'))
to_ret.append(jentry)
return to_ret
def add_to_stream_log_cache(self, cacheKey, item):
rKey = self.prefix_redis_key+cacheKey

9
helpers/trendings_helper.py
View File

@ -26,11 +26,16 @@ class Trendings_helper:
#logger
logDir = cfg.get('Log', 'directory')
logfilename = cfg.get('Log', 'filename')
logfilename = cfg.get('Log', 'helpers_filename')
logPath = os.path.join(logDir, logfilename)
if not os.path.exists(logDir):
os.makedirs(logDir)
logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
try:
logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
except PermissionError as error:
print(error)
print("Please fix the above and try again.")
sys.exit(126)
self.logger = logging.getLogger(__name__)
''' SETTER '''

9
helpers/users_helper.py
View File

@ -23,11 +23,16 @@ class Users_helper:
#logger
logDir = cfg.get('Log', 'directory')
logfilename = cfg.get('Log', 'filename')
logfilename = cfg.get('Log', 'helpers_filename')
logPath = os.path.join(logDir, logfilename)
if not os.path.exists(logDir):
os.makedirs(logDir)
logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
try:
logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
except PermissionError as error:
print(error)
print("Please fix the above and try again.")
sys.exit(126)
self.logger = logging.getLogger(__name__)
def add_user_login(self, timestamp, org, email=''):

22
install_dependencies.sh
View File

@ -1,12 +1,21 @@
#!/bin/bash
set -e
## disable -e for production systems
#set -e
## Debug mode
#set -x
sudo apt-get install python3-virtualenv virtualenv screen redis-server unzip -y
if [ -z "$VIRTUAL_ENV" ]; then
virtualenv -p python3 DASHENV
virtualenv -p python3 DASHENV ; DASH_VENV=$?
if [[ "$DASH_VENV" != "0" ]]; then
echo "Something went wrong with either the update or install of the virtualenv."
echo "Please investigate manually."
exit $DASH_VENV
fi
. ./DASHENV/bin/activate
fi
@ -35,7 +44,14 @@ mkdir -p css fonts js
popd
mkdir -p temp
wget http://www.misp-project.org/assets/images/misp-small.png -O static/pics/MISP.png
NET_WGET=$(wget --no-cache -q https://www.misp-project.org/assets/images/misp-small.png -O static/pics/MISP.png; echo $?)
if [[ "$NET_WGET" != "0" ]]; then
echo "The first wget we tried failed, please investigate manually."
exit $NET_WGET
fi
wget https://www.misp-project.org/favicon.ico -O static/favicon.ico
# jquery
JQVERSION="3.2.1"

6
server.py
View File

@ -14,7 +14,7 @@ import redis
import util
from flask import (Flask, Response, jsonify, render_template, request,
stream_with_context)
send_from_directory, stream_with_context)
from helpers import (contributor_helper, geo_helper, live_helper,
trendings_helper, users_helper)
@ -141,6 +141,10 @@ def index():
zoomlevel=cfg.getint('Dashboard' ,'zoomlevel')
)
@app.route('/favicon.ico')
def favicon():
return send_from_directory(os.path.join(app.root_path, 'static'),
'favicon.ico', mimetype='image/vnd.microsoft.icon')
@app.route("/geo")
def geo():

14
zmq_dispatcher.py
View File

@ -23,11 +23,16 @@ cfg = configparser.ConfigParser()
cfg.read(configfile)
logDir = cfg.get('Log', 'directory')
logfilename = cfg.get('Log', 'filename')
logfilename = cfg.get('Log', 'dispatcher_filename')
logPath = os.path.join(logDir, logfilename)
if not os.path.exists(logDir):
os.makedirs(logDir)
logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
try:
logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
except PermissionError as error:
print(error)
print("Please fix the above and try again.")
sys.exit(126)
logger = logging.getLogger('zmq_dispatcher')
LISTNAME = cfg.get('RedisLIST', 'listName')
@ -289,4 +294,7 @@ if __name__ == "__main__":
parser.add_argument('-s', '--sleep', required=False, dest='sleeptime', type=int, help='The number of second to wait before checking redis list size', default=5)
args = parser.parse_args()
main(args.sleeptime)
try:
main(args.sleeptime)
except (redis.exceptions.ResponseError, KeyboardInterrupt) as error:
print(error)

16
zmq_subscriber.py
View File

@ -16,11 +16,16 @@ configfile = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'config/c
cfg = configparser.ConfigParser()
cfg.read(configfile)
logDir = cfg.get('Log', 'directory')
logfilename = cfg.get('Log', 'filename')
logfilename = cfg.get('Log', 'subscriber_filename')
logPath = os.path.join(logDir, logfilename)
if not os.path.exists(logDir):
os.makedirs(logDir)
logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
try:
logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
except PermissionError as error:
print(error)
print("Please fix the above and try again.")
sys.exit(126)
logger = logging.getLogger('zmq_subscriber')
ZMQ_URL = cfg.get('RedisGlobal', 'zmq_url')
@ -59,9 +64,12 @@ def main(zmqName):
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='A zmq subscriber. It subscribes to a ZNQ then redispatch it to the misp-dashboard')
parser = argparse.ArgumentParser(description='A zmq subscriber. It subscribes to a ZMQ then redispatch it to the misp-dashboard')
parser.add_argument('-n', '--name', required=False, dest='zmqname', help='The ZMQ feed name', default="MISP Standard ZMQ")
parser.add_argument('-u', '--url', required=False, dest='zmqurl', help='The URL to connect to', default=ZMQ_URL)
args = parser.parse_args()
main(args.zmqname)
try:
main(args.zmqname)
except redis.exceptions.ResponseError as error:
print(error)