Merge pull request #183 from schutzwerk/email-notifications

New feature: E-Mail notifications for tracked terms
pull/184/head
Sami Mokaddem 2018-03-01 09:56:38 +01:00 committed by GitHub
commit e00c9f0f45
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 292 additions and 78 deletions

View File

@ -35,6 +35,9 @@ import calendar
from Helper import Process
# Email notifications
from NotificationHelper import *
# Config Variables
BlackListTermsSet_Name = "BlackListSetTermSet"
TrackedTermsSet_Name = "TrackedSetTermSet"
@ -45,7 +48,6 @@ top_termFreq_setName_week = ["TopTermFreq_set_week", 7]
top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month]
def check_if_tracked_term(term, path):
if term in server_term.smembers(TrackedTermsSet_Name):
#add_paste to tracked_word_set
@ -54,6 +56,13 @@ def check_if_tracked_term(term, path):
print term, 'addded', set_name, '->', path
p.populate_set_out("New Term added", 'CurveManageTopSets')
# Send a notification only when the member is in the set
if term in server_term.smembers(TrackedTermsNotificationEnabled_Name):
# Send to every associated email adress
for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + term):
sendEmailNotification(email, term)
def getValueOverRange(word, startDate, num_day):
to_return = 0

75
bin/NotificationHelper.py Executable file
View File

@ -0,0 +1,75 @@
#!/usr/bin/env python2
# -*-coding:UTF-8 -*
import ConfigParser
import os
import smtplib
from email.MIMEMultipart import MIMEMultipart
from email.mime.text import MIMEText
"""
This module allows the global configuration and management of notification settings and methods.
"""
# CONFIG #
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
# notifications enabled/disabled
TrackedTermsNotificationEnabled_Name = "TrackedNotifications"
# associated notification email addresses for a specific term`
# Keys will be e.g. TrackedNotificationEmails<TERMNAME>
TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_"
def sendEmailNotification(recipient, term):
if not os.path.exists(configfile):
raise Exception('Unable to find the configuration file. \
Did you set environment variables? \
Or activate the virtualenv?')
cfg = ConfigParser.ConfigParser()
cfg.read(configfile)
sender = cfg.get("Notifications", "sender"),
sender_host = cfg.get("Notifications", "sender_host"),
sender_port = cfg.getint("Notifications", "sender_port"),
sender_pw = cfg.get("Notifications", "sender_pw"),
if isinstance(sender, tuple):
sender = sender[0]
if isinstance(sender_host, tuple):
sender_host = sender_host[0]
if isinstance(sender_port, tuple):
sender_port = sender_port[0]
if isinstance(sender_pw, tuple):
sender_pw = sender_pw[0]
if (
sender is not None and
sender_host is not None and
sender_port is not None and
sender_pw is not None
):
try:
server_ssl = smtplib.SMTP_SSL(sender_host, sender_port)
server_ssl.ehlo()
server_ssl.login(sender, sender_pw)
mime_msg = MIMEMultipart()
mime_msg['From'] = sender
mime_msg['To'] = recipient
mime_msg['Subject'] = "AIL Term Alert"
body = "New occurrence for term: " + term
mime_msg.attach(MIMEText(body, 'plain'))
server_ssl.sendmail(sender, recipient, mime_msg.as_string())
server_ssl.quit()
except Exception as e:
print str(e)

View File

@ -12,18 +12,22 @@ from pubsublogger import publisher
from packages import lib_words
from packages import Paste
import os
from os import environ
import datetime
import calendar
import re
from Helper import Process
# Email notifications
from NotificationHelper import *
# Config Variables
DICO_REFRESH_TIME = 60 #s
BlackListTermsSet_Name = "BlackListSetTermSet"
TrackedTermsSet_Name = "TrackedSetTermSet"
TrackedRegexSet_Name = "TrackedRegexSet"
top_term_freq_max_set_cardinality = 20 # Max cardinality of the terms frequences set
oneDay = 60*60*24
top_termFreq_setName_day = ["TopTermFreq_set_day_", 1]
@ -86,8 +90,15 @@ if __name__ == "__main__":
if matched is not None: #there is a match
print('regex matched {}'.format(regex_str))
matched = matched.group(0)
regex_str_complete = "/" + regex_str + "/"
# Add in Regex track set only if term is not in the blacklist
if matched not in server_term.smembers(BlackListTermsSet_Name):
if regex_str_complete not in server_term.smembers(BlackListTermsSet_Name):
# Send a notification only when the member is in the set
if regex_str_complete in server_term.smembers(TrackedTermsNotificationEnabled_Name):
# Send to every associated email adress
for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + regex_str_complete):
sendEmailNotification(email, regex_str)
set_name = 'regex_' + dico_regexname_to_redis[regex_str]
new_to_the_set = server_term.sadd(set_name, filename)
new_to_the_set = True if new_to_the_set == 1 else False

View File

@ -16,14 +16,17 @@ import datetime
import calendar
import re
import ast
from Helper import Process
# Email notifications
from NotificationHelper import *
# Config Variables
BlackListTermsSet_Name = "BlackListSetTermSet"
TrackedTermsSet_Name = "TrackedSetTermSet"
TrackedRegexSet_Name = "TrackedRegexSet"
TrackedSetSet_Name = "TrackedSetSet"
top_term_freq_max_set_cardinality = 20 # Max cardinality of the terms frequences set
oneDay = 60*60*24
top_termFreq_setName_day = ["TopTermFreq_set_day_", 1]
@ -72,7 +75,6 @@ if __name__ == "__main__":
else:
continue
message = p.get_from_set()
while True:
@ -100,12 +102,17 @@ if __name__ == "__main__":
for the_set, matchingNum in match_dico.items():
eff_percent = float(matchingNum) / float((len(ast.literal_eval(the_set))-1)) * 100 #-1 bc if the percent matching
if eff_percent >= dico_percent[the_set]:
# Send a notification only when the member is in the set
if dico_setname_to_redis[str(the_set)] in server_term.smembers(TrackedTermsNotificationEnabled_Name):
# Send to every associated email adress
for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + dico_setname_to_redis[str(the_set)]):
sendEmailNotification(email, dico_setname_to_redis[str(the_set)])
print(the_set, "matched in", filename)
set_name = 'set_' + dico_setname_to_redis[the_set]
new_to_the_set = server_term.sadd(set_name, filename)
new_to_the_set = True if new_to_the_set == 1 else False
#consider the num of occurence of this set
set_value = int(server_term.hincrby(timestamp, dico_setname_to_redis[the_set], int(1)))

View File

@ -18,6 +18,13 @@ pystemonpath = /home/pystemon/pystemon/
sentiment_lexicon_file = sentiment/vader_lexicon.zip/vader_lexicon/vader_lexicon.txt
##### Notifications ######
[Notifications]
sender = sender@example.com
sender_host = smtp.example.com
sender_port = 1337
sender_pw = securepassword
##### Flask #####
[Flask]
#Maximum number of character to display in the toolip

View File

@ -42,6 +42,14 @@ TrackedRegexDate_Name = "TrackedRegexDate"
TrackedSetSet_Name = "TrackedSetSet"
TrackedSetDate_Name = "TrackedSetDate"
# notifications enabled/disabled
# same value as in `bin/NotificationHelper.py`
TrackedTermsNotificationEnabled_Name = "TrackedNotifications"
# associated notification email addresses for a specific term`
# same value as in `bin/NotificationHelper.py`
# Keys will be e.g. TrackedNotificationEmails_<TERMNAME>
TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_"
'''CRED'''
REGEX_CRED = '[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+'
@ -138,11 +146,23 @@ def terms_management():
today = today.replace(hour=0, minute=0, second=0, microsecond=0)
today_timestamp = calendar.timegm(today.timetuple())
# Map tracking if notifications are enabled for a specific term
notificationEnabledDict = {}
# Maps a specific term to the associated email addresses
notificationEMailTermMapping = {}
#Regex
trackReg_list = []
trackReg_list_values = []
trackReg_list_num_of_paste = []
for tracked_regex in r_serv_term.smembers(TrackedRegexSet_Name):
notificationEMailTermMapping[tracked_regex] = "\n".join(r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_regex))
if tracked_regex not in notificationEnabledDict:
notificationEnabledDict[tracked_regex] = False
trackReg_list.append(tracked_regex)
value_range = Term_getValueOverRange(tracked_regex, today_timestamp, [1, 7, 31], per_paste=per_paste_text)
@ -154,11 +174,21 @@ def terms_management():
value_range.append(term_date)
trackReg_list_values.append(value_range)
if tracked_regex in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name):
notificationEnabledDict[tracked_regex] = True
#Set
trackSet_list = []
trackSet_list_values = []
trackSet_list_num_of_paste = []
for tracked_set in r_serv_term.smembers(TrackedSetSet_Name):
notificationEMailTermMapping[tracked_set] = "\n".join(r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_set))
if tracked_set not in notificationEnabledDict:
notificationEnabledDict[tracked_set] = False
trackSet_list.append(tracked_set)
value_range = Term_getValueOverRange(tracked_set, today_timestamp, [1, 7, 31], per_paste=per_paste_text)
@ -170,11 +200,20 @@ def terms_management():
value_range.append(term_date)
trackSet_list_values.append(value_range)
if tracked_set in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name):
notificationEnabledDict[tracked_set] = True
#Tracked terms
track_list = []
track_list_values = []
track_list_num_of_paste = []
for tracked_term in r_serv_term.smembers(TrackedTermsSet_Name):
notificationEMailTermMapping[tracked_term] = "\n".join(r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_term))
if tracked_term not in notificationEnabledDict:
notificationEnabledDict[tracked_term] = False
track_list.append(tracked_term)
value_range = Term_getValueOverRange(tracked_term, today_timestamp, [1, 7, 31], per_paste=per_paste_text)
@ -186,6 +225,8 @@ def terms_management():
value_range.append(term_date)
track_list_values.append(value_range)
if tracked_term in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name):
notificationEnabledDict[tracked_term] = True
#blacklist terms
black_list = []
@ -199,7 +240,7 @@ def terms_management():
track_list_values=track_list_values, track_list_num_of_paste=track_list_num_of_paste,
trackReg_list_values=trackReg_list_values, trackReg_list_num_of_paste=trackReg_list_num_of_paste,
trackSet_list_values=trackSet_list_values, trackSet_list_num_of_paste=trackSet_list_num_of_paste,
per_paste=per_paste)
per_paste=per_paste, notificationEnabledDict=notificationEnabledDict, notificationEMailTermMapping=notificationEMailTermMapping)
@terms.route("/terms_management_query_paste/")
@ -267,16 +308,39 @@ def terms_management_action():
section = request.args.get('section')
action = request.args.get('action')
term = request.args.get('term')
notificationEmailsParam = request.args.get('emailAddresses')
if action is None or term is None:
return "None"
else:
if section == "followTerm":
if action == "add":
# Strip all whitespace
notificationEmailsParam = "".join(notificationEmailsParam.split())
# Make a list of all passed email addresses
notificationEmails = notificationEmailsParam.split(",")
validNotificationEmails = []
# check for valid email addresses
for email in notificationEmails:
# Really basic validation:
# has exactly one @ sign, and at least one . in the part after the @
if re.match(r"[^@]+@[^@]+\.[^@]+", email):
validNotificationEmails.append(email)
# check if regex/set or simple term
#regex
if term.startswith('/') and term.endswith('/'):
r_serv_term.sadd(TrackedRegexSet_Name, term)
r_serv_term.hset(TrackedRegexDate_Name, term, today_timestamp)
# add all valid emails to the set
for email in validNotificationEmails:
r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term, email)
# enable notifications by default
r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term)
#set
elif term.startswith('\\') and term.endswith('\\'):
@ -290,11 +354,31 @@ def terms_management_action():
set_to_add = "\\" + tab_term[:-1] + ", [{}]]\\".format(match_percent)
r_serv_term.sadd(TrackedSetSet_Name, set_to_add)
r_serv_term.hset(TrackedSetDate_Name, set_to_add, today_timestamp)
# add all valid emails to the set
for email in validNotificationEmails:
r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + set_to_add, email)
# enable notifications by default
r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, set_to_add)
#simple term
else:
r_serv_term.sadd(TrackedTermsSet_Name, term.lower())
r_serv_term.hset(TrackedTermsDate_Name, term.lower(), today_timestamp)
# add all valid emails to the set
for email in validNotificationEmails:
r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term.lower(), email)
# enable notifications by default
r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term.lower())
elif action == "toggleEMailNotification":
# get the current state
if term in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name):
# remove it
r_serv_term.srem(TrackedTermsNotificationEnabled_Name, term.lower())
else:
# add it
r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term.lower())
#del action
else:
if term.startswith('/') and term.endswith('/'):
@ -308,6 +392,9 @@ def terms_management_action():
r_serv_term.srem(TrackedTermsSet_Name, term.lower())
r_serv_term.hdel(TrackedTermsDate_Name, term.lower())
# delete the associated notification emails too
r_serv_term.delete(TrackedTermsNotificationEmailsPrefix_Name + term)
elif section == "blacklistTerm":
if action == "add":
r_serv_term.sadd(BlackListTermsSet_Name, term.lower())

View File

@ -97,6 +97,7 @@
<div class="form-group input-group" style="margin-bottom: 30px;">
<span class="input-group-addon"><span class="fa fa-eye"></span></span>
<input id="followTermInput" class="form-control" placeholder="Term to track." type="text" style="max-width: 400px;">
<input id="followTermEMailNotificationReceiversInput" class="form-control" placeholder="Notification E-Mails (comma separated)" type="text" style="max-width: 400px;">
<button id="followTermBtn" class="btn btn-success btn-interaction" style="margin-left: 10px;" data-section="followTerm" data-action="add"> Add term</button>
</div>
@ -110,6 +111,7 @@
<th>Month occurence</th>
<th># tracked paste</th>
<th>Action</th>
<th>Notification E-Mails</th>
</tr>
</thead>
<tbody>
@ -122,10 +124,12 @@
<td>{{ trackSet_list_values[loop.index0][1] }}</td>
<td>{{ trackSet_list_values[loop.index0][2] }}</td>
<td>{{ trackSet_list_num_of_paste[loop.index0] }}</td>
<td><p style="margin: 0px;">
<td><p style="margin: 0px; white-space: nowrap;">
<span data-toggle="modal" data-target="#mymodal" data-term="{{ set }}" ><button class="btn-link" data-toggle="tooltip" data-placement="right" title="Show concerned paste(s)"><span class="glyphicon glyphicon-info-sign"></span></button></span>
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ set }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
&nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ set }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[set] %} checked {% endif %}>
</p></td>
<td style="white-space:pre">{{ notificationEMailTermMapping[set] }}</td>
</tr>
{% endfor %}
<!-- REGEX -->
@ -137,10 +141,12 @@
<td>{{ trackReg_list_values[loop.index0][1] }}</td>
<td>{{ trackReg_list_values[loop.index0][2] }}</td>
<td>{{ trackReg_list_num_of_paste[loop.index0] }}</td>
<td><p style="margin: 0px;">
<td><p style="margin: 0px; white-space: nowrap;">
<span data-toggle="modal" data-target="#mymodal" data-term="{{ regex }}" ><button class="btn-link" data-toggle="tooltip" data-placement="right" title="Show concerned paste(s)"><span class="glyphicon glyphicon-info-sign"></span></button></span>
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ regex }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
&nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ regex }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[regex] %} checked {% endif %}>
</p></td>
<td style="white-space:pre">{{ notificationEMailTermMapping[regex] }}</td>
</tr>
{% endfor %}
<!-- Normal term -->
@ -152,10 +158,12 @@
<td>{{ track_list_values[loop.index0][1] }}</td>
<td>{{ track_list_values[loop.index0][2] }}</td>
<td>{{ track_list_num_of_paste[loop.index0] }}</td>
<td><p style="margin: 0px;">
<td><p style="margin: 0px; white-space: nowrap;">
<span data-toggle="modal" data-target="#mymodal" data-term="{{ term }}" ><button class="btn-link" data-toggle="tooltip" data-placement="right" title="Show concerned paste(s)"><span class="glyphicon glyphicon-info-sign"></span></button></span>
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ term }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
&nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ term }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[term] %} checked {% endif %}>
</p></td>
<td style="white-space:pre">{{ notificationEMailTermMapping[term] }}</td>
</tr>
{% endfor %}
</tbody>
@ -231,43 +239,9 @@
var table_track;
var table_black;
$(document).ready(function(){
activePage = $('h1.page-header').attr('data-page');
$("#"+activePage).addClass("active");
if({{ per_paste }} == 1) {
$("#per_paste").attr('checked', true)
}
$('[data-toggle="tooltip"]').tooltip();
table_track = $('#myTable').DataTable();
table_black = $('#myTable2').DataTable();
table_track.on( 'draw.dt', function () {
perform_binding();
});
table_black.on( 'draw.dt', function () {
perform_binding();
});
$("#followTermInput").keyup(function(event){
if(event.keyCode == 13){
$("#followTermBtn").click();
$("#followTermInput").val("");
}
});
$("#blacklistTermInput").keyup(function(event){
if(event.keyCode == 13){
$("#blacklistTermBtn").click();
$("#blacklistTermInput").val("");
}
});
perform_binding();
function bindEventsForCurrentPage() {
// On click, get html content from url and update the corresponding modal
$("[data-toggle='modal']").on("click.openmodal", function (event) {
$("[data-toggle='modal']").unbind().on("click.openmodal", function (event) {
//console.log(data);
event.preventDefault();
var the_modal=$(this);
@ -315,6 +289,45 @@
});
}
$(document).ready(function(){
bindEventsForCurrentPage();
activePage = $('h1.page-header').attr('data-page');
$("#"+activePage).addClass("active");
if({{ per_paste }} == 1) {
$("#per_paste").attr('checked', true)
}
$('[data-toggle="tooltip"]').tooltip();
table_track = $('#myTable').DataTable();
table_black = $('#myTable2').DataTable();
table_track.on( 'draw.dt', function () {
perform_binding();
});
table_black.on( 'draw.dt', function () {
perform_binding();
});
$("#followTermInput").keyup(function(event){
if(event.keyCode == 13){
$("#followTermBtn").click();
$("#followTermInput").val("");
}
});
$("#blacklistTermInput").keyup(function(event){
if(event.keyCode == 13){
$("#blacklistTermBtn").click();
$("#blacklistTermInput").val("");
}
});
perform_binding();
$("#mymodal").on('hidden.bs.modal', function () {
$("#mymodalbody").html("<p>Loading paste information...</p>");
var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_for('static', filename='image/loading.gif') }}\" height='26' width='26' style='margin: 4px;'>";
@ -339,10 +352,12 @@ function perform_operation(){
var curr_action = $(this).attr('data-action');
if (curr_action == "add") {
var curr_term = $('#'+curr_section+'Input').val();
var email_addresses = $('#followTermEMailNotificationReceiversInput').val();
} else {
var curr_term = $(this).attr('data-content');
var email_addresses = "";
}
var data_to_send = { section: curr_section, action:curr_action, term: curr_term};
var data_to_send = { section: curr_section, action: curr_action, term: curr_term, emailAddresses: email_addresses};
if (curr_term != "") {
console.log(data_to_send);
@ -354,8 +369,11 @@ function perform_operation(){
if(json.action == "add") {
// query data
$.get("{{ url_for('terms.terms_management_query') }}", { term: json.term, section: json.section }, function(data2, status){
var action_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"left\" title=\"Remove this term\" data-content=\"" + json.term + "\" data-section=\"followTerm\" data-action=\"delete\"><span class=\"glyphicon glyphicon-trash\"></span></button>"
table_track.row.add( [ json.term, data2[3], data2[0], data2[1], data2[2], 0, action_button ] ).draw( false );
var delete_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"left\" title=\"Remove this term\" data-content=\"" + json.term + "\" data-section=\"followTerm\" data-action=\"delete\"><span class=\"glyphicon glyphicon-trash\"></span></button>";
var info_button = "<span data-toggle=\"modal\" data-target=\"#mymodal\" data-term=\"" + json.term + "\" ><button class=\"btn-link\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Show concerned paste(s)\"><span class=\"glyphicon glyphicon-info-sign\"></span></button></span>";
var enabled_checkbox = "<input id=\"checkBoxEMailAlerts\" type=\"checkbox\" title=\"Toggle E-Mail notifications\" class=\"btn-link btn-interaction\" data-content=\"" + json.term + "\" data-section=\"followTerm\" data-action=\"toggleEMailNotification\" checked>";
var action_buttons = "<p style=\"margin: 0px; white-space: nowrap;\">" + info_button + delete_button + " &nbsp; " + enabled_checkbox + "</p>";
table_track.row.add( [ json.term, data2[3], data2[0], data2[1], data2[2], 0, action_buttons, $('#followTermEMailNotificationReceiversInput').val().replace(",", "\n") ] ).draw( false );
perform_binding();
});
} else if (json.action == "delete") {