mirror of https://github.com/MISP/misp-dashboard
chg: [log] Added 3 seperate log files: helpers.log / zmq_subscribre.log / zmq_dispatcher.log
fix: [log] Catch permission errors on log filespull/97/head
parent
6b291a5526
commit
92b91c5c05
|
@ -33,7 +33,9 @@ additional_help_text = ["Sightings multiplies earned points by 2", "Editing an a
|
||||||
|
|
||||||
[Log]
|
[Log]
|
||||||
directory=logs
|
directory=logs
|
||||||
filename=logs.log
|
dispatcher_filename=zmq_dispatcher.log
|
||||||
|
subscriber_filename=zmq_subscriber.log
|
||||||
|
helpers_filename=helpers.log
|
||||||
|
|
||||||
[RedisGlobal]
|
[RedisGlobal]
|
||||||
host=localhost
|
host=localhost
|
||||||
|
|
|
@ -33,11 +33,16 @@ class Contributor_helper:
|
||||||
|
|
||||||
#logger
|
#logger
|
||||||
logDir = cfg.get('Log', 'directory')
|
logDir = cfg.get('Log', 'directory')
|
||||||
logfilename = cfg.get('Log', 'filename')
|
logfilename = cfg.get('Log', 'helpers_filename')
|
||||||
logPath = os.path.join(logDir, logfilename)
|
logPath = os.path.join(logDir, logfilename)
|
||||||
if not os.path.exists(logDir):
|
if not os.path.exists(logDir):
|
||||||
os.makedirs(logDir)
|
os.makedirs(logDir)
|
||||||
logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
|
try:
|
||||||
|
logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
|
||||||
|
except PermissionError as error:
|
||||||
|
print(error)
|
||||||
|
print("Please fix the above and try again.")
|
||||||
|
sys.exit(126)
|
||||||
self.logger = logging.getLogger(__name__)
|
self.logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
#honorBadge
|
#honorBadge
|
||||||
|
@ -123,14 +128,14 @@ class Contributor_helper:
|
||||||
if action in ['edit', None]:
|
if action in ['edit', None]:
|
||||||
pass
|
pass
|
||||||
#return #not a contribution?
|
#return #not a contribution?
|
||||||
|
|
||||||
now = datetime.datetime.now()
|
now = datetime.datetime.now()
|
||||||
nowSec = int(time.time())
|
nowSec = int(time.time())
|
||||||
pnts_to_add = self.default_pnts_per_contribution
|
pnts_to_add = self.default_pnts_per_contribution
|
||||||
|
|
||||||
# Do not consider contribution as login anymore
|
# Do not consider contribution as login anymore
|
||||||
#self.users_helper.add_user_login(nowSec, org)
|
#self.users_helper.add_user_login(nowSec, org)
|
||||||
|
|
||||||
# is a valid contribution
|
# is a valid contribution
|
||||||
if categ is not None:
|
if categ is not None:
|
||||||
try:
|
try:
|
||||||
|
@ -138,23 +143,23 @@ class Contributor_helper:
|
||||||
except KeyError:
|
except KeyError:
|
||||||
pnts_to_add = self.default_pnts_per_contribution
|
pnts_to_add = self.default_pnts_per_contribution
|
||||||
pnts_to_add *= pntMultiplier
|
pnts_to_add *= pntMultiplier
|
||||||
|
|
||||||
util.push_to_redis_zset(self.serv_redis_db, self.keyDay, org, count=pnts_to_add)
|
util.push_to_redis_zset(self.serv_redis_db, self.keyDay, org, count=pnts_to_add)
|
||||||
#CONTRIB_CATEG retain the contribution per category, not the point earned in this categ
|
#CONTRIB_CATEG retain the contribution per category, not the point earned in this categ
|
||||||
util.push_to_redis_zset(self.serv_redis_db, self.keyCateg, org, count=1, endSubkey=':'+util.noSpaceLower(categ))
|
util.push_to_redis_zset(self.serv_redis_db, self.keyCateg, org, count=1, endSubkey=':'+util.noSpaceLower(categ))
|
||||||
self.publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'categ': categ, 'action': action, 'epoch': nowSec }, channel=self.CHANNEL_LASTCONTRIB)
|
self.publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'categ': categ, 'action': action, 'epoch': nowSec }, channel=self.CHANNEL_LASTCONTRIB)
|
||||||
else:
|
else:
|
||||||
categ = ""
|
categ = ""
|
||||||
|
|
||||||
self.serv_redis_db.sadd(self.keyAllOrg, org)
|
self.serv_redis_db.sadd(self.keyAllOrg, org)
|
||||||
|
|
||||||
keyname = "{}:{}".format(self.keyLastContrib, util.getDateStrFormat(now))
|
keyname = "{}:{}".format(self.keyLastContrib, util.getDateStrFormat(now))
|
||||||
self.serv_redis_db.zadd(keyname, nowSec, org)
|
self.serv_redis_db.zadd(keyname, nowSec, org)
|
||||||
self.logger.debug('Added to redis: keyname={}, nowSec={}, org={}'.format(keyname, nowSec, org))
|
self.logger.debug('Added to redis: keyname={}, nowSec={}, org={}'.format(keyname, nowSec, org))
|
||||||
self.serv_redis_db.expire(keyname, util.ONE_DAY*7) #expire after 7 day
|
self.serv_redis_db.expire(keyname, util.ONE_DAY*7) #expire after 7 day
|
||||||
|
|
||||||
awards_given = self.updateOrgContributionRank(org, pnts_to_add, action, contribType, eventTime=datetime.datetime.now(), isLabeled=isLabeled, categ=util.noSpaceLower(categ))
|
awards_given = self.updateOrgContributionRank(org, pnts_to_add, action, contribType, eventTime=datetime.datetime.now(), isLabeled=isLabeled, categ=util.noSpaceLower(categ))
|
||||||
|
|
||||||
for award in awards_given:
|
for award in awards_given:
|
||||||
# update awards given
|
# update awards given
|
||||||
keyname = "{}:{}".format(self.keyLastAward, util.getDateStrFormat(now))
|
keyname = "{}:{}".format(self.keyLastAward, util.getDateStrFormat(now))
|
||||||
|
|
|
@ -32,11 +32,16 @@ class Geo_helper:
|
||||||
|
|
||||||
#logger
|
#logger
|
||||||
logDir = cfg.get('Log', 'directory')
|
logDir = cfg.get('Log', 'directory')
|
||||||
logfilename = cfg.get('Log', 'filename')
|
logfilename = cfg.get('Log', 'helpers_filename')
|
||||||
logPath = os.path.join(logDir, logfilename)
|
logPath = os.path.join(logDir, logfilename)
|
||||||
if not os.path.exists(logDir):
|
if not os.path.exists(logDir):
|
||||||
os.makedirs(logDir)
|
os.makedirs(logDir)
|
||||||
logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
|
try:
|
||||||
|
logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
|
||||||
|
except PermissionError as error:
|
||||||
|
print(error)
|
||||||
|
print("Please fix the above and try again.")
|
||||||
|
sys.exit(126)
|
||||||
self.logger = logging.getLogger(__name__)
|
self.logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
self.keyCategCoord = "GEO_COORD"
|
self.keyCategCoord = "GEO_COORD"
|
||||||
|
|
|
@ -17,11 +17,16 @@ class Live_helper:
|
||||||
|
|
||||||
# logger
|
# logger
|
||||||
logDir = cfg.get('Log', 'directory')
|
logDir = cfg.get('Log', 'directory')
|
||||||
logfilename = cfg.get('Log', 'filename')
|
logfilename = cfg.get('Log', 'helpers_filename')
|
||||||
logPath = os.path.join(logDir, logfilename)
|
logPath = os.path.join(logDir, logfilename)
|
||||||
if not os.path.exists(logDir):
|
if not os.path.exists(logDir):
|
||||||
os.makedirs(logDir)
|
os.makedirs(logDir)
|
||||||
logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
|
try:
|
||||||
|
logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
|
||||||
|
except PermissionError as error:
|
||||||
|
print(error)
|
||||||
|
print("Please fix the above and try again.")
|
||||||
|
sys.exit(126)
|
||||||
self.logger = logging.getLogger(__name__)
|
self.logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
def publish_log(self, zmq_name, name, content, channel=None):
|
def publish_log(self, zmq_name, name, content, channel=None):
|
||||||
|
@ -44,7 +49,7 @@ class Live_helper:
|
||||||
jentry = json.loads(entry.decode('utf8'))
|
jentry = json.loads(entry.decode('utf8'))
|
||||||
to_ret.append(jentry)
|
to_ret.append(jentry)
|
||||||
return to_ret
|
return to_ret
|
||||||
|
|
||||||
|
|
||||||
def add_to_stream_log_cache(self, cacheKey, item):
|
def add_to_stream_log_cache(self, cacheKey, item):
|
||||||
rKey = self.prefix_redis_key+cacheKey
|
rKey = self.prefix_redis_key+cacheKey
|
||||||
|
|
|
@ -26,11 +26,16 @@ class Trendings_helper:
|
||||||
|
|
||||||
#logger
|
#logger
|
||||||
logDir = cfg.get('Log', 'directory')
|
logDir = cfg.get('Log', 'directory')
|
||||||
logfilename = cfg.get('Log', 'filename')
|
logfilename = cfg.get('Log', 'helpers_filename')
|
||||||
logPath = os.path.join(logDir, logfilename)
|
logPath = os.path.join(logDir, logfilename)
|
||||||
if not os.path.exists(logDir):
|
if not os.path.exists(logDir):
|
||||||
os.makedirs(logDir)
|
os.makedirs(logDir)
|
||||||
logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
|
try:
|
||||||
|
logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
|
||||||
|
except PermissionError as error:
|
||||||
|
print(error)
|
||||||
|
print("Please fix the above and try again.")
|
||||||
|
sys.exit(126)
|
||||||
self.logger = logging.getLogger(__name__)
|
self.logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
''' SETTER '''
|
''' SETTER '''
|
||||||
|
|
|
@ -23,11 +23,16 @@ class Users_helper:
|
||||||
|
|
||||||
#logger
|
#logger
|
||||||
logDir = cfg.get('Log', 'directory')
|
logDir = cfg.get('Log', 'directory')
|
||||||
logfilename = cfg.get('Log', 'filename')
|
logfilename = cfg.get('Log', 'helpers_filename')
|
||||||
logPath = os.path.join(logDir, logfilename)
|
logPath = os.path.join(logDir, logfilename)
|
||||||
if not os.path.exists(logDir):
|
if not os.path.exists(logDir):
|
||||||
os.makedirs(logDir)
|
os.makedirs(logDir)
|
||||||
logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
|
try:
|
||||||
|
logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
|
||||||
|
except PermissionError as error:
|
||||||
|
print(error)
|
||||||
|
print("Please fix the above and try again.")
|
||||||
|
sys.exit(126)
|
||||||
self.logger = logging.getLogger(__name__)
|
self.logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
def add_user_login(self, timestamp, org, email=''):
|
def add_user_login(self, timestamp, org, email=''):
|
||||||
|
|
|
@ -23,7 +23,7 @@ cfg = configparser.ConfigParser()
|
||||||
cfg.read(configfile)
|
cfg.read(configfile)
|
||||||
|
|
||||||
logDir = cfg.get('Log', 'directory')
|
logDir = cfg.get('Log', 'directory')
|
||||||
logfilename = cfg.get('Log', 'filename')
|
logfilename = cfg.get('Log', 'dispatcher_filename')
|
||||||
logPath = os.path.join(logDir, logfilename)
|
logPath = os.path.join(logDir, logfilename)
|
||||||
if not os.path.exists(logDir):
|
if not os.path.exists(logDir):
|
||||||
os.makedirs(logDir)
|
os.makedirs(logDir)
|
||||||
|
@ -296,5 +296,5 @@ if __name__ == "__main__":
|
||||||
|
|
||||||
try:
|
try:
|
||||||
main(args.sleeptime)
|
main(args.sleeptime)
|
||||||
except redis.exceptions.ResponseError as error:
|
except (redis.exceptions.ResponseError, KeyboardInterrupt) as error:
|
||||||
print(error)
|
print(error)
|
||||||
|
|
|
@ -16,7 +16,7 @@ configfile = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'config/c
|
||||||
cfg = configparser.ConfigParser()
|
cfg = configparser.ConfigParser()
|
||||||
cfg.read(configfile)
|
cfg.read(configfile)
|
||||||
logDir = cfg.get('Log', 'directory')
|
logDir = cfg.get('Log', 'directory')
|
||||||
logfilename = cfg.get('Log', 'filename')
|
logfilename = cfg.get('Log', 'subscriber_filename')
|
||||||
logPath = os.path.join(logDir, logfilename)
|
logPath = os.path.join(logDir, logfilename)
|
||||||
if not os.path.exists(logDir):
|
if not os.path.exists(logDir):
|
||||||
os.makedirs(logDir)
|
os.makedirs(logDir)
|
||||||
|
@ -64,7 +64,7 @@ def main(zmqName):
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(description='A zmq subscriber. It subscribes to a ZNQ then redispatch it to the misp-dashboard')
|
parser = argparse.ArgumentParser(description='A zmq subscriber. It subscribes to a ZMQ then redispatch it to the misp-dashboard')
|
||||||
parser.add_argument('-n', '--name', required=False, dest='zmqname', help='The ZMQ feed name', default="MISP Standard ZMQ")
|
parser.add_argument('-n', '--name', required=False, dest='zmqname', help='The ZMQ feed name', default="MISP Standard ZMQ")
|
||||||
parser.add_argument('-u', '--url', required=False, dest='zmqurl', help='The URL to connect to', default=ZMQ_URL)
|
parser.add_argument('-u', '--url', required=False, dest='zmqurl', help='The URL to connect to', default=ZMQ_URL)
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
Loading…
Reference in New Issue