mirror of https://github.com/MISP/misp-dashboard
Added logger in config
parent
bdef0c21e9
commit
9139ed7eef
|
@ -11,6 +11,9 @@ item_to_plot = Attribute.category
|
||||||
size_dashboard_left_width = 5
|
size_dashboard_left_width = 5
|
||||||
size_openStreet_pannel_perc = 55
|
size_openStreet_pannel_perc = 55
|
||||||
size_world_pannel_perc = 35
|
size_world_pannel_perc = 35
|
||||||
|
field_to_plot = Attribute.category
|
||||||
|
fieldname_order=["Event.id", "Attribute.Tag", "Attribute.category", "Attribute.type", ["Attribute.value", "Attribute.comment"]]
|
||||||
|
char_separator=||
|
||||||
|
|
||||||
[GEO]
|
[GEO]
|
||||||
#min
|
#min
|
||||||
|
@ -25,9 +28,8 @@ min_between_reload = 5
|
||||||
additional_help_text = ["Sightings multiplies earned points by 2", "Editing an attribute earns you the same as creating one"]
|
additional_help_text = ["Sightings multiplies earned points by 2", "Editing an attribute earns you the same as creating one"]
|
||||||
|
|
||||||
[Log]
|
[Log]
|
||||||
field_to_plot = Attribute.category
|
directory=logs
|
||||||
fieldname_order=["Event.id", "Attribute.Tag", "Attribute.category", "Attribute.type", ["Attribute.value", "Attribute.comment"]]
|
filename=logs.log
|
||||||
char_separator=||
|
|
||||||
|
|
||||||
[RedisGlobal]
|
[RedisGlobal]
|
||||||
host=localhost
|
host=localhost
|
||||||
|
|
|
@ -12,8 +12,6 @@ import redis
|
||||||
import util
|
import util
|
||||||
from . import users_helper
|
from . import users_helper
|
||||||
KEYDAY = "CONTRIB_DAY" # To be used by other module
|
KEYDAY = "CONTRIB_DAY" # To be used by other module
|
||||||
logging.basicConfig(filename='logs/logs.log', filemode='w', level=logging.INFO)
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
class Contributor_helper:
|
class Contributor_helper:
|
||||||
def __init__(self, serv_redis_db, cfg):
|
def __init__(self, serv_redis_db, cfg):
|
||||||
|
@ -29,6 +27,15 @@ class Contributor_helper:
|
||||||
self.CHANNEL_LASTCONTRIB = cfg.get('RedisLog', 'channelLastContributor')
|
self.CHANNEL_LASTCONTRIB = cfg.get('RedisLog', 'channelLastContributor')
|
||||||
self.users_helper = users_helper.Users_helper(serv_redis_db, cfg)
|
self.users_helper = users_helper.Users_helper(serv_redis_db, cfg)
|
||||||
|
|
||||||
|
#logger
|
||||||
|
logDir = cfg.get('Log', 'directory')
|
||||||
|
logfilename = cfg.get('Log', 'filename')
|
||||||
|
logPath = os.path.join(logDir, logfilename)
|
||||||
|
if not os.path.exists(logDir):
|
||||||
|
os.makedirs(logDir)
|
||||||
|
logging.basicConfig(filename=logPath, filemode='w', level=logging.INFO)
|
||||||
|
self.logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
#honorBadge
|
#honorBadge
|
||||||
self.honorBadgeNum = len(self.cfg_org_rank.options('HonorBadge'))
|
self.honorBadgeNum = len(self.cfg_org_rank.options('HonorBadge'))
|
||||||
self.heavilyCount = self.cfg_org_rank.getint('rankRequirementsMisc', 'heavilyCount')
|
self.heavilyCount = self.cfg_org_rank.getint('rankRequirementsMisc', 'heavilyCount')
|
||||||
|
@ -98,12 +105,12 @@ class Contributor_helper:
|
||||||
today_str = util.getDateStrFormat(date)
|
today_str = util.getDateStrFormat(date)
|
||||||
keyname = "{}:{}:{}".format(self.keyCateg, today_str, categ)
|
keyname = "{}:{}:{}".format(self.keyCateg, today_str, categ)
|
||||||
self.serv_redis_db.zincrby(keyname, org, count)
|
self.serv_redis_db.zincrby(keyname, org, count)
|
||||||
logger.debug('Added to redis: keyname={}, org={}, count={}'.format(keyname, org, count))
|
self.logger.debug('Added to redis: keyname={}, org={}, count={}'.format(keyname, org, count))
|
||||||
|
|
||||||
def publish_log(self, zmq_name, name, content, channel=""):
|
def publish_log(self, zmq_name, name, content, channel=""):
|
||||||
to_send = { 'name': name, 'log': json.dumps(content), 'zmqName': zmq_name }
|
to_send = { 'name': name, 'log': json.dumps(content), 'zmqName': zmq_name }
|
||||||
self.serv_log.publish(channel, json.dumps(to_send))
|
self.serv_log.publish(channel, json.dumps(to_send))
|
||||||
logger.debug('Published: {}'.format(json.dumps(to_send)))
|
self.logger.debug('Published: {}'.format(json.dumps(to_send)))
|
||||||
|
|
||||||
''' HANDLER '''
|
''' HANDLER '''
|
||||||
#pntMultiplier if one contribution rewards more than others. (e.g. shighting may gives more points than editing)
|
#pntMultiplier if one contribution rewards more than others. (e.g. shighting may gives more points than editing)
|
||||||
|
@ -138,7 +145,7 @@ class Contributor_helper:
|
||||||
|
|
||||||
keyname = "{}:{}".format(self.keyLastContrib, util.getDateStrFormat(now))
|
keyname = "{}:{}".format(self.keyLastContrib, util.getDateStrFormat(now))
|
||||||
self.serv_redis_db.zadd(keyname, nowSec, org)
|
self.serv_redis_db.zadd(keyname, nowSec, org)
|
||||||
logger.debug('Added to redis: keyname={}, nowSec={}, org={}'.format(keyname, nowSec, org))
|
self.logger.debug('Added to redis: keyname={}, nowSec={}, org={}'.format(keyname, nowSec, org))
|
||||||
self.serv_redis_db.expire(keyname, util.ONE_DAY*7) #expire after 7 day
|
self.serv_redis_db.expire(keyname, util.ONE_DAY*7) #expire after 7 day
|
||||||
|
|
||||||
awards_given = self.updateOrgContributionRank(org, pnts_to_add, action, contribType, eventTime=datetime.datetime.now(), isLabeled=isLabeled, categ=util.noSpaceLower(categ))
|
awards_given = self.updateOrgContributionRank(org, pnts_to_add, action, contribType, eventTime=datetime.datetime.now(), isLabeled=isLabeled, categ=util.noSpaceLower(categ))
|
||||||
|
@ -147,7 +154,7 @@ class Contributor_helper:
|
||||||
# update awards given
|
# update awards given
|
||||||
keyname = "{}:{}".format(self.keyLastAward, util.getDateStrFormat(now))
|
keyname = "{}:{}".format(self.keyLastAward, util.getDateStrFormat(now))
|
||||||
self.serv_redis_db.zadd(keyname, nowSec, json.dumps({'org': org, 'award': award, 'epoch': nowSec }))
|
self.serv_redis_db.zadd(keyname, nowSec, json.dumps({'org': org, 'award': award, 'epoch': nowSec }))
|
||||||
logger.debug('Added to redis: keyname={}, nowSec={}, content={}'.format(keyname, nowSec, json.dumps({'org': org, 'award': award, 'epoch': nowSec })))
|
self.logger.debug('Added to redis: keyname={}, nowSec={}, content={}'.format(keyname, nowSec, json.dumps({'org': org, 'award': award, 'epoch': nowSec })))
|
||||||
self.serv_redis_db.expire(keyname, util.ONE_DAY*7) #expire after 7 day
|
self.serv_redis_db.expire(keyname, util.ONE_DAY*7) #expire after 7 day
|
||||||
# publish
|
# publish
|
||||||
self.publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'award': award, 'epoch': nowSec }, channel=self.CHANNEL_LASTAWARDS)
|
self.publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'award': award, 'epoch': nowSec }, channel=self.CHANNEL_LASTAWARDS)
|
||||||
|
@ -221,17 +228,17 @@ class Contributor_helper:
|
||||||
if contribType == 'Attribute':
|
if contribType == 'Attribute':
|
||||||
attributeWeekCount = self.serv_redis_db.incrby(keyname.format(org=orgName, orgCateg='ATTR_WEEK_COUNT'), 1)
|
attributeWeekCount = self.serv_redis_db.incrby(keyname.format(org=orgName, orgCateg='ATTR_WEEK_COUNT'), 1)
|
||||||
self.serv_redis_db.expire(keyname.format(org=orgName, orgCateg='ATTR_WEEK_COUNT'), util.ONE_DAY*7)
|
self.serv_redis_db.expire(keyname.format(org=orgName, orgCateg='ATTR_WEEK_COUNT'), util.ONE_DAY*7)
|
||||||
logger.debug('Incrby: keyname={}'.format(keyname.format(org=orgName, orgCateg='ATTR_WEEK_COUNT')))
|
self.logger.debug('Incrby: keyname={}'.format(keyname.format(org=orgName, orgCateg='ATTR_WEEK_COUNT')))
|
||||||
|
|
||||||
if contribType == 'Proposal':
|
if contribType == 'Proposal':
|
||||||
proposalWeekCount = self.serv_redis_db.incrby(keyname.format(org=orgName, orgCateg='PROP_WEEK_COUNT'), 1)
|
proposalWeekCount = self.serv_redis_db.incrby(keyname.format(org=orgName, orgCateg='PROP_WEEK_COUNT'), 1)
|
||||||
logger.debug('Incrby: keyname={}'.format(keyname.format(org=orgName, orgCateg='PROP_WEEK_COUNT')))
|
self.logger.debug('Incrby: keyname={}'.format(keyname.format(org=orgName, orgCateg='PROP_WEEK_COUNT')))
|
||||||
self.serv_redis_db.expire(keyname.format(org=orgName, orgCateg='PROP_WEEK_COUNT'), util.ONE_DAY*7)
|
self.serv_redis_db.expire(keyname.format(org=orgName, orgCateg='PROP_WEEK_COUNT'), util.ONE_DAY*7)
|
||||||
addContributionToCateg(datetime.datetime.now(), 'proposal')
|
addContributionToCateg(datetime.datetime.now(), 'proposal')
|
||||||
|
|
||||||
if contribType == 'Sighting':
|
if contribType == 'Sighting':
|
||||||
sightingWeekCount = self.serv_redis_db.incrby(keyname.format(org=orgName, orgCateg='SIGHT_WEEK_COUNT'), 1)
|
sightingWeekCount = self.serv_redis_db.incrby(keyname.format(org=orgName, orgCateg='SIGHT_WEEK_COUNT'), 1)
|
||||||
logger.debug('Incrby: keyname={}'.format(keyname.format(org=orgName, orgCateg='SIGHT_WEEK_COUNT')))
|
self.logger.debug('Incrby: keyname={}'.format(keyname.format(org=orgName, orgCateg='SIGHT_WEEK_COUNT')))
|
||||||
self.serv_redis_db.expire(keyname.format(org=orgName, orgCateg='SIGHT_WEEK_COUNT'), util.ONE_DAY*7)
|
self.serv_redis_db.expire(keyname.format(org=orgName, orgCateg='SIGHT_WEEK_COUNT'), util.ONE_DAY*7)
|
||||||
self.addContributionToCateg(datetime.datetime.now(), 'sighting', orgName)
|
self.addContributionToCateg(datetime.datetime.now(), 'sighting', orgName)
|
||||||
|
|
||||||
|
@ -240,11 +247,11 @@ class Contributor_helper:
|
||||||
|
|
||||||
if contribType == 'Event':
|
if contribType == 'Event':
|
||||||
eventWeekCount = self.serv_redis_db.incrby(keyname.format(org=orgName, orgCateg='EVENT_WEEK_COUNT'), 1)
|
eventWeekCount = self.serv_redis_db.incrby(keyname.format(org=orgName, orgCateg='EVENT_WEEK_COUNT'), 1)
|
||||||
logger.debug('Incrby: keyname={}'.format(keyname.format(org=orgName, orgCateg='EVENT_WEEK_COUNT')))
|
self.logger.debug('Incrby: keyname={}'.format(keyname.format(org=orgName, orgCateg='EVENT_WEEK_COUNT')))
|
||||||
self.serv_redis_db.expire(keyname.format(org=orgName, orgCateg='EVENT_WEEK_COUNT'), util.ONE_DAY*7)
|
self.serv_redis_db.expire(keyname.format(org=orgName, orgCateg='EVENT_WEEK_COUNT'), util.ONE_DAY*7)
|
||||||
|
|
||||||
eventMonthCount = self.serv_redis_db.incrby(keyname.format(org=orgName, orgCateg='EVENT_MONTH_COUNT'), 1)
|
eventMonthCount = self.serv_redis_db.incrby(keyname.format(org=orgName, orgCateg='EVENT_MONTH_COUNT'), 1)
|
||||||
logger.debug('Incrby: keyname={}'.format(keyname.format(org=orgName, orgCateg='EVENT_MONTH_COUNT')))
|
self.logger.debug('Incrby: keyname={}'.format(keyname.format(org=orgName, orgCateg='EVENT_MONTH_COUNT')))
|
||||||
self.serv_redis_db.expire(keyname.format(org=orgName, orgCateg='EVENT_MONTH_COUNT'), util.ONE_DAY*7)
|
self.serv_redis_db.expire(keyname.format(org=orgName, orgCateg='EVENT_MONTH_COUNT'), util.ONE_DAY*7)
|
||||||
|
|
||||||
# getRequirement parameters
|
# getRequirement parameters
|
||||||
|
@ -287,7 +294,7 @@ class Contributor_helper:
|
||||||
|
|
||||||
for rankReq, ttl in contrib:
|
for rankReq, ttl in contrib:
|
||||||
self.serv_redis_db.set(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankReq)), 1)
|
self.serv_redis_db.set(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankReq)), 1)
|
||||||
logger.debug('Set: keyname={}'.format(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankReq))))
|
self.logger.debug('Set: keyname={}'.format(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankReq))))
|
||||||
self.serv_redis_db.expire(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankReq)), ttl)
|
self.serv_redis_db.expire(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankReq)), ttl)
|
||||||
|
|
||||||
ContributionStatus = self.getCurrentContributionStatus(orgName)
|
ContributionStatus = self.getCurrentContributionStatus(orgName)
|
||||||
|
@ -335,12 +342,12 @@ class Contributor_helper:
|
||||||
def giveBadgeToOrg(self, org, badgeNum):
|
def giveBadgeToOrg(self, org, badgeNum):
|
||||||
keyname = '{mainKey}:{org}:{orgCateg}'
|
keyname = '{mainKey}:{org}:{orgCateg}'
|
||||||
self.serv_redis_db.set(keyname.format(mainKey=self.keyContribReq, org=org, orgCateg='BADGE_'+str(badgeNum)), 1)
|
self.serv_redis_db.set(keyname.format(mainKey=self.keyContribReq, org=org, orgCateg='BADGE_'+str(badgeNum)), 1)
|
||||||
logger.debug('Giving badge {} to org {}'.format(org, badgeNum))
|
self.logger.debug('Giving badge {} to org {}'.format(org, badgeNum))
|
||||||
|
|
||||||
def removeBadgeFromOrg(self, org, badgeNum):
|
def removeBadgeFromOrg(self, org, badgeNum):
|
||||||
keyname = '{mainKey}:{org}:{orgCateg}'
|
keyname = '{mainKey}:{org}:{orgCateg}'
|
||||||
self.serv_redis_db.delete(keyname.format(mainKey=self.keyContribReq, org=org, orgCateg='BADGE_'+str(badgeNum)))
|
self.serv_redis_db.delete(keyname.format(mainKey=self.keyContribReq, org=org, orgCateg='BADGE_'+str(badgeNum)))
|
||||||
logger.debug('Removing badge {} from org {}'.format(org, badgeNum))
|
self.logger.debug('Removing badge {} from org {}'.format(org, badgeNum))
|
||||||
|
|
||||||
''' TROPHIES '''
|
''' TROPHIES '''
|
||||||
def getOrgTrophies(self, org):
|
def getOrgTrophies(self, org):
|
||||||
|
@ -391,12 +398,12 @@ class Contributor_helper:
|
||||||
def giveTrophyPointsToOrg(self, org, categ, points):
|
def giveTrophyPointsToOrg(self, org, categ, points):
|
||||||
keyname = '{mainKey}:{orgCateg}'
|
keyname = '{mainKey}:{orgCateg}'
|
||||||
self.serv_redis_db.zincrby(keyname.format(mainKey=self.keyTrophy, orgCateg=categ), org, points)
|
self.serv_redis_db.zincrby(keyname.format(mainKey=self.keyTrophy, orgCateg=categ), org, points)
|
||||||
logger.debug('Giving {} trophy points to {} in {}'.format(points, org, categ))
|
self.logger.debug('Giving {} trophy points to {} in {}'.format(points, org, categ))
|
||||||
|
|
||||||
def removeTrophyPointsFromOrg(self, org, categ, points):
|
def removeTrophyPointsFromOrg(self, org, categ, points):
|
||||||
keyname = '{mainKey}:{orgCateg}'
|
keyname = '{mainKey}:{orgCateg}'
|
||||||
self.serv_redis_db.zincrby(keyname.format(mainKey=self.keyTrophy, orgCateg=categ), org, -points)
|
self.serv_redis_db.zincrby(keyname.format(mainKey=self.keyTrophy, orgCateg=categ), org, -points)
|
||||||
logger.debug('Removing {} trophy points from {} in {}'.format(points, org, categ))
|
self.logger.debug('Removing {} trophy points from {} in {}'.format(points, org, categ))
|
||||||
|
|
||||||
''' AWARDS HELPER '''
|
''' AWARDS HELPER '''
|
||||||
def getLastAwardsFromRedis(self):
|
def getLastAwardsFromRedis(self):
|
||||||
|
|
|
@ -13,9 +13,6 @@ from phonenumbers import geocoder
|
||||||
|
|
||||||
import util
|
import util
|
||||||
|
|
||||||
logging.basicConfig(filename='logs/logs.log', filemode='w', level=logging.INFO)
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
class Geo_helper:
|
class Geo_helper:
|
||||||
def __init__(self, serv_redis_db, cfg):
|
def __init__(self, serv_redis_db, cfg):
|
||||||
self.serv_redis_db = serv_redis_db
|
self.serv_redis_db = serv_redis_db
|
||||||
|
@ -25,6 +22,15 @@ class Geo_helper:
|
||||||
port=cfg.getint('RedisGlobal', 'port'),
|
port=cfg.getint('RedisGlobal', 'port'),
|
||||||
db=cfg.getint('RedisMap', 'db'))
|
db=cfg.getint('RedisMap', 'db'))
|
||||||
|
|
||||||
|
#logger
|
||||||
|
logDir = cfg.get('Log', 'directory')
|
||||||
|
logfilename = cfg.get('Log', 'filename')
|
||||||
|
logPath = os.path.join(logDir, logfilename)
|
||||||
|
if not os.path.exists(logDir):
|
||||||
|
os.makedirs(logDir)
|
||||||
|
logging.basicConfig(filename=logPath, filemode='w', level=logging.INFO)
|
||||||
|
self.logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
self.keyCategCoord = "GEO_COORD"
|
self.keyCategCoord = "GEO_COORD"
|
||||||
self.keyCategCountry = "GEO_COUNTRY"
|
self.keyCategCountry = "GEO_COUNTRY"
|
||||||
self.keyCategRad = "GEO_RAD"
|
self.keyCategRad = "GEO_RAD"
|
||||||
|
@ -106,22 +112,22 @@ class Geo_helper:
|
||||||
"regionCode": rep['full_rep'].country.iso_code,
|
"regionCode": rep['full_rep'].country.iso_code,
|
||||||
}
|
}
|
||||||
self.serv_coord.publish(self.CHANNELDISP, json.dumps(to_send))
|
self.serv_coord.publish(self.CHANNELDISP, json.dumps(to_send))
|
||||||
logger.info('Published: {}'.format(json.dumps(to_send)))
|
self.logger.info('Published: {}'.format(json.dumps(to_send)))
|
||||||
except ValueError:
|
except ValueError:
|
||||||
logger.warning("can't resolve ip")
|
self.logger.warning("can't resolve ip")
|
||||||
except geoip2.errors.AddressNotFoundError:
|
except geoip2.errors.AddressNotFoundError:
|
||||||
logger.warning("Address not in Database")
|
self.logger.warning("Address not in Database")
|
||||||
|
|
||||||
def getCoordFromPhoneAndPublish(self, phoneNumber, categ):
|
def getCoordFromPhoneAndPublish(self, phoneNumber, categ):
|
||||||
try:
|
try:
|
||||||
rep = phonenumbers.parse(phoneNumber, None)
|
rep = phonenumbers.parse(phoneNumber, None)
|
||||||
if not (phonenumbers.is_valid_number(rep) or phonenumbers.is_possible_number(rep)):
|
if not (phonenumbers.is_valid_number(rep) or phonenumbers.is_possible_number(rep)):
|
||||||
logger.warning("Phone number not valid")
|
self.logger.warning("Phone number not valid")
|
||||||
return
|
return
|
||||||
country_name = geocoder.country_name_for_number(rep, "en")
|
country_name = geocoder.country_name_for_number(rep, "en")
|
||||||
country_code = self.country_to_iso[country_name]
|
country_code = self.country_to_iso[country_name]
|
||||||
if country_code is None:
|
if country_code is None:
|
||||||
logger.warning("Non matching ISO_CODE")
|
self.logger.warning("Non matching ISO_CODE")
|
||||||
return
|
return
|
||||||
coord = self.country_code_to_coord[country_code.lower()] # countrycode is in upper case
|
coord = self.country_code_to_coord[country_code.lower()] # countrycode is in upper case
|
||||||
coord_dic = {'lat': coord['lat'], 'lon': coord['long']}
|
coord_dic = {'lat': coord['lat'], 'lon': coord['long']}
|
||||||
|
@ -146,9 +152,9 @@ class Geo_helper:
|
||||||
"regionCode": country_code,
|
"regionCode": country_code,
|
||||||
}
|
}
|
||||||
self.serv_coord.publish(self.CHANNELDISP, json.dumps(to_send))
|
self.serv_coord.publish(self.CHANNELDISP, json.dumps(to_send))
|
||||||
logger.info('Published: {}'.format(json.dumps(to_send)))
|
self.logger.info('Published: {}'.format(json.dumps(to_send)))
|
||||||
except phonenumbers.NumberParseException:
|
except phonenumbers.NumberParseException:
|
||||||
logger.warning("Can't resolve phone number country")
|
self.logger.warning("Can't resolve phone number country")
|
||||||
|
|
||||||
''' UTIL '''
|
''' UTIL '''
|
||||||
def push_to_redis_geo(self, keyCateg, lon, lat, content):
|
def push_to_redis_geo(self, keyCateg, lon, lat, content):
|
||||||
|
@ -156,13 +162,13 @@ class Geo_helper:
|
||||||
today_str = util.getDateStrFormat(now)
|
today_str = util.getDateStrFormat(now)
|
||||||
keyname = "{}:{}".format(keyCateg, today_str)
|
keyname = "{}:{}".format(keyCateg, today_str)
|
||||||
self.serv_redis_db.geoadd(keyname, lon, lat, content)
|
self.serv_redis_db.geoadd(keyname, lon, lat, content)
|
||||||
logger.debug('Added to redis: keyname={}, lon={}, lat={}, content={}'.format(keyname, lon, lat, content))
|
self.logger.debug('Added to redis: keyname={}, lon={}, lat={}, content={}'.format(keyname, lon, lat, content))
|
||||||
def push_to_redis_zset(self, keyCateg, toAdd, endSubkey="", count=1):
|
def push_to_redis_zset(self, keyCateg, toAdd, endSubkey="", count=1):
|
||||||
now = datetime.datetime.now()
|
now = datetime.datetime.now()
|
||||||
today_str = util.getDateStrFormat(now)
|
today_str = util.getDateStrFormat(now)
|
||||||
keyname = "{}:{}{}".format(keyCateg, today_str, endSubkey)
|
keyname = "{}:{}{}".format(keyCateg, today_str, endSubkey)
|
||||||
self.serv_redis_db.zincrby(keyname, toAdd, count)
|
self.serv_redis_db.zincrby(keyname, toAdd, count)
|
||||||
logger.debug('Added to redis: keyname={}, toAdd={}, count={}'.format(keyname, toAdd, count))
|
self.logger.debug('Added to redis: keyname={}, toAdd={}, count={}'.format(keyname, toAdd, count))
|
||||||
|
|
||||||
def ip_to_coord(self, ip):
|
def ip_to_coord(self, ip):
|
||||||
resp = self.reader.city(ip)
|
resp = self.reader.city(ip)
|
||||||
|
|
|
@ -7,9 +7,6 @@ from collections import OrderedDict
|
||||||
|
|
||||||
import util
|
import util
|
||||||
|
|
||||||
logging.basicConfig(filename='logs/logs.log', filemode='w', level=logging.INFO)
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
class Trendings_helper:
|
class Trendings_helper:
|
||||||
def __init__(self, serv_redis_db, cfg):
|
def __init__(self, serv_redis_db, cfg):
|
||||||
self.serv_redis_db = serv_redis_db
|
self.serv_redis_db = serv_redis_db
|
||||||
|
@ -23,6 +20,15 @@ class Trendings_helper:
|
||||||
self.keySigh = "TRENDINGS_SIGHT_sightings"
|
self.keySigh = "TRENDINGS_SIGHT_sightings"
|
||||||
self.keyFalse = "TRENDINGS_SIGHT_false_positive"
|
self.keyFalse = "TRENDINGS_SIGHT_false_positive"
|
||||||
|
|
||||||
|
#logger
|
||||||
|
logDir = cfg.get('Log', 'directory')
|
||||||
|
logfilename = cfg.get('Log', 'filename')
|
||||||
|
logPath = os.path.join(logDir, logfilename)
|
||||||
|
if not os.path.exists(logDir):
|
||||||
|
os.makedirs(logDir)
|
||||||
|
logging.basicConfig(filename=logPath, filemode='w', level=logging.INFO)
|
||||||
|
self.logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
''' SETTER '''
|
''' SETTER '''
|
||||||
|
|
||||||
def addGenericTrending(self, trendingType, data, timestamp):
|
def addGenericTrending(self, trendingType, data, timestamp):
|
||||||
|
@ -34,7 +40,7 @@ class Trendings_helper:
|
||||||
else:
|
else:
|
||||||
to_save = data
|
to_save = data
|
||||||
self.serv_redis_db.zincrby(keyname, to_save, 1)
|
self.serv_redis_db.zincrby(keyname, to_save, 1)
|
||||||
logger.debug('Added to redis: keyname={}, content={}'.format(keyname, to_save))
|
self.logger.debug('Added to redis: keyname={}, content={}'.format(keyname, to_save))
|
||||||
|
|
||||||
def addTrendingEvent(self, eventName, timestamp):
|
def addTrendingEvent(self, eventName, timestamp):
|
||||||
self.addGenericTrending(self.keyEvent, eventName, timestamp)
|
self.addGenericTrending(self.keyEvent, eventName, timestamp)
|
||||||
|
@ -58,14 +64,14 @@ class Trendings_helper:
|
||||||
timestampDate_str = util.getDateStrFormat(timestampDate)
|
timestampDate_str = util.getDateStrFormat(timestampDate)
|
||||||
keyname = "{}:{}".format(self.keySigh, timestampDate_str)
|
keyname = "{}:{}".format(self.keySigh, timestampDate_str)
|
||||||
self.serv_redis_db.incrby(keyname, 1)
|
self.serv_redis_db.incrby(keyname, 1)
|
||||||
logger.debug('Incrby: keyname={}'.format(keyname))
|
self.logger.debug('Incrby: keyname={}'.format(keyname))
|
||||||
|
|
||||||
def addFalsePositive(self, timestamp):
|
def addFalsePositive(self, timestamp):
|
||||||
timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
|
timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
|
||||||
timestampDate_str = util.getDateStrFormat(timestampDate)
|
timestampDate_str = util.getDateStrFormat(timestampDate)
|
||||||
keyname = "{}:{}".format(self.keyFalse, timestampDate_str)
|
keyname = "{}:{}".format(self.keyFalse, timestampDate_str)
|
||||||
self.serv_redis_db.incrby(keyname, 1)
|
self.serv_redis_db.incrby(keyname, 1)
|
||||||
logger.debug('Incrby: keyname={}'.format(keyname))
|
self.logger.debug('Incrby: keyname={}'.format(keyname))
|
||||||
|
|
||||||
''' GETTER '''
|
''' GETTER '''
|
||||||
|
|
||||||
|
|
|
@ -7,8 +7,6 @@ import logging
|
||||||
import util
|
import util
|
||||||
from . import contributor_helper
|
from . import contributor_helper
|
||||||
|
|
||||||
logging.basicConfig(filename='logs/logs.log', filemode='w', level=logging.INFO)
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
class Users_helper:
|
class Users_helper:
|
||||||
def __init__(self, serv_redis_db, cfg):
|
def __init__(self, serv_redis_db, cfg):
|
||||||
|
@ -20,12 +18,21 @@ class Users_helper:
|
||||||
self.keyOrgLog = "LOGIN_ORG"
|
self.keyOrgLog = "LOGIN_ORG"
|
||||||
self.keyContribDay = contributor_helper.KEYDAY # Key to get monthly contribution
|
self.keyContribDay = contributor_helper.KEYDAY # Key to get monthly contribution
|
||||||
|
|
||||||
|
#logger
|
||||||
|
logDir = cfg.get('Log', 'directory')
|
||||||
|
logfilename = cfg.get('Log', 'filename')
|
||||||
|
logPath = os.path.join(logDir, logfilename)
|
||||||
|
if not os.path.exists(logDir):
|
||||||
|
os.makedirs(logDir)
|
||||||
|
logging.basicConfig(filename=logPath, filemode='w', level=logging.INFO)
|
||||||
|
self.logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
def addTemporary(self, org, timestamp):
|
def addTemporary(self, org, timestamp):
|
||||||
timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
|
timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
|
||||||
timestampDate_str = util.getDateHoursStrFormat(timestampDate)
|
timestampDate_str = util.getDateHoursStrFormat(timestampDate)
|
||||||
keyname_timestamp = "{}:{}".format(self.keyTimestampSet, timestampDate_str)
|
keyname_timestamp = "{}:{}".format(self.keyTimestampSet, timestampDate_str)
|
||||||
self.serv_redis_db.sadd(keyname_timestamp, org)
|
self.serv_redis_db.sadd(keyname_timestamp, org)
|
||||||
logger.debug('Added to redis: keyname={}, org={}'.format(keyname_timestamp, org))
|
self.logger.debug('Added to redis: keyname={}, org={}'.format(keyname_timestamp, org))
|
||||||
self.serv_redis_db.expire(keyname_timestamp, 60*60)
|
self.serv_redis_db.expire(keyname_timestamp, 60*60)
|
||||||
|
|
||||||
def hasAlreadyBeenAdded(self, org, timestamp):
|
def hasAlreadyBeenAdded(self, org, timestamp):
|
||||||
|
@ -44,12 +51,12 @@ class Users_helper:
|
||||||
if not self.hasAlreadyBeenAdded(org, timestamp):
|
if not self.hasAlreadyBeenAdded(org, timestamp):
|
||||||
keyname_timestamp = "{}:{}".format(self.keyTimestamp, timestampDate_str)
|
keyname_timestamp = "{}:{}".format(self.keyTimestamp, timestampDate_str)
|
||||||
self.serv_redis_db.sadd(keyname_timestamp, timestamp)
|
self.serv_redis_db.sadd(keyname_timestamp, timestamp)
|
||||||
logger.debug('Added to redis: keyname={}, org={}'.format(keyname_timestamp, timestamp))
|
self.logger.debug('Added to redis: keyname={}, org={}'.format(keyname_timestamp, timestamp))
|
||||||
self.addTemporary(org, timestamp)
|
self.addTemporary(org, timestamp)
|
||||||
|
|
||||||
keyname_org = "{}:{}".format(self.keyOrgLog, timestampDate_str)
|
keyname_org = "{}:{}".format(self.keyOrgLog, timestampDate_str)
|
||||||
self.serv_redis_db.zincrby(keyname_org, org, 1)
|
self.serv_redis_db.zincrby(keyname_org, org, 1)
|
||||||
logger.debug('Added to redis: keyname={}, org={}'.format(keyname_org, org))
|
self.logger.debug('Added to redis: keyname={}, org={}'.format(keyname_org, org))
|
||||||
|
|
||||||
def getUserLogins(self, date):
|
def getUserLogins(self, date):
|
||||||
keyname = "{}:{}".format(self.keyTimestamp, util.getDateStrFormat(date))
|
keyname = "{}:{}".format(self.keyTimestamp, util.getDateStrFormat(date))
|
||||||
|
|
|
@ -65,7 +65,7 @@ class LogItem():
|
||||||
FIELDNAME_ORDER_HEADER = []
|
FIELDNAME_ORDER_HEADER = []
|
||||||
FIELDNAME_ORDER.append("Time")
|
FIELDNAME_ORDER.append("Time")
|
||||||
FIELDNAME_ORDER_HEADER.append("Time")
|
FIELDNAME_ORDER_HEADER.append("Time")
|
||||||
for item in json.loads(cfg.get('Log', 'fieldname_order')):
|
for item in json.loads(cfg.get('Dashboard', 'fieldname_order')):
|
||||||
if type(item) is list:
|
if type(item) is list:
|
||||||
FIELDNAME_ORDER_HEADER.append(" | ".join(item))
|
FIELDNAME_ORDER_HEADER.append(" | ".join(item))
|
||||||
else:
|
else:
|
||||||
|
|
|
@ -22,7 +22,12 @@ configfile = os.path.join(os.environ['DASH_CONFIG'], 'config.cfg')
|
||||||
cfg = configparser.ConfigParser()
|
cfg = configparser.ConfigParser()
|
||||||
cfg.read(configfile)
|
cfg.read(configfile)
|
||||||
|
|
||||||
logging.basicConfig(filename='logs/logs.log', filemode='w', level=logging.INFO)
|
logDir = cfg.get('Log', 'directory')
|
||||||
|
logfilename = cfg.get('Log', 'filename')
|
||||||
|
logPath = os.path.join(logDir, logfilename)
|
||||||
|
if not os.path.exists(logDir):
|
||||||
|
os.makedirs(logDir)
|
||||||
|
logging.basicConfig(filename=logPath, filemode='w', level=logging.INFO)
|
||||||
logger = logging.getLogger('zmq_dispatcher')
|
logger = logging.getLogger('zmq_dispatcher')
|
||||||
|
|
||||||
CHANNEL = cfg.get('RedisLog', 'channel')
|
CHANNEL = cfg.get('RedisLog', 'channel')
|
||||||
|
@ -193,7 +198,7 @@ def handler_attribute(zmq_name, jsonobj, hasAlreadyBeenContributed=False):
|
||||||
trendings_helper.addTrendingTags(tags, timestamp)
|
trendings_helper.addTrendingTags(tags, timestamp)
|
||||||
|
|
||||||
to_push = []
|
to_push = []
|
||||||
for field in json.loads(cfg.get('Log', 'fieldname_order')):
|
for field in json.loads(cfg.get('Dashboard', 'fieldname_order')):
|
||||||
if type(field) is list:
|
if type(field) is list:
|
||||||
to_join = []
|
to_join = []
|
||||||
for subField in field:
|
for subField in field:
|
||||||
|
|
|
@ -13,8 +13,12 @@ import json
|
||||||
configfile = os.path.join(os.environ['DASH_CONFIG'], 'config.cfg')
|
configfile = os.path.join(os.environ['DASH_CONFIG'], 'config.cfg')
|
||||||
cfg = configparser.ConfigParser()
|
cfg = configparser.ConfigParser()
|
||||||
cfg.read(configfile)
|
cfg.read(configfile)
|
||||||
|
logDir = cfg.get('Log', 'directory')
|
||||||
logging.basicConfig(filename='logs/logs.log', filemode='w', level=logging.INFO)
|
logfilename = cfg.get('Log', 'filename')
|
||||||
|
logPath = os.path.join(logDir, logfilename)
|
||||||
|
if not os.path.exists(logDir):
|
||||||
|
os.makedirs(logDir)
|
||||||
|
logging.basicConfig(filename=logPath, filemode='w', level=logging.INFO)
|
||||||
logger = logging.getLogger('zmq_subscriber')
|
logger = logging.getLogger('zmq_subscriber')
|
||||||
|
|
||||||
ZMQ_URL = cfg.get('RedisGlobal', 'zmq_url')
|
ZMQ_URL = cfg.get('RedisGlobal', 'zmq_url')
|
||||||
|
|
Loading…
Reference in New Issue