CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

Fixed e-mail notificatins for sets and regexes

pull/183/head
Philipp Schmied 2018-02-28 11:31:16 +01:00
parent b6fcf51309
commit 29608ab29b
4 changed files with 31 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
bin/RegexForTermsFrequency.py
View File

@ -37,7 +37,7 @@ top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, to
def refresh_dicos():
dico_regex = {}
dico_regex = {}
dico_regexname_to_redis = {}
for regex_str in server_term.smembers(TrackedRegexSet_Name):
dico_regex[regex_str[1:-1]] = re.compile(regex_str[1:-1])
@ -90,18 +90,15 @@ if __name__ == "__main__":
if matched is not None: #there is a match
print('regex matched {}'.format(regex_str))
matched = matched.group(0)
regex_str_complete = "/" + regex_str + "/"
# Add in Regex track set only if term is not in the blacklist
if matched not in server_term.smembers(BlackListTermsSet_Name):
if regex_str_complete not in server_term.smembers(BlackListTermsSet_Name):
# Send a notification only when the member is in the set
if matched in server_term.smembers(TrackedTermsNotificationEnabled_Name):
if regex_str_complete in server_term.smembers(TrackedTermsNotificationEnabled_Name):
# Send to every associated email adress
for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + matched):
sendEmailNotification(email, matched)
for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + regex_str_complete):
sendEmailNotification(email, regex_str)
set_name = 'regex_' + dico_regexname_to_redis[regex_str]
new_to_the_set = server_term.sadd(set_name, filename)
new_to_the_set = True if new_to_the_set == 1 else False

13
bin/SetForTermsFrequency.py
View File

@ -75,7 +75,6 @@ if __name__ == "__main__":
else:
continue
message = p.get_from_set()
while True:
@ -103,19 +102,17 @@ if __name__ == "__main__":
for the_set, matchingNum in match_dico.items():
eff_percent = float(matchingNum) / float((len(ast.literal_eval(the_set))-1)) * 100 #-1 bc if the percent matching
if eff_percent >= dico_percent[the_set]:
# Send a notification only when the member is in the set
if the_set in server_term.smembers(TrackedTermsNotificationEnabled_Name):
if dico_setname_to_redis[str(the_set)] in server_term.smembers(TrackedTermsNotificationEnabled_Name):
# Send to every associated email adress
for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + the_set):
sendEmailNotification(email, the_set)
for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + dico_setname_to_redis[str(the_set)]):
sendEmailNotification(email, dico_setname_to_redis[str(the_set)])
print(the_set, "matched in", filename)
set_name = 'set_' + dico_setname_to_redis[the_set]
new_to_the_set = server_term.sadd(set_name, filename)
new_to_the_set = True if new_to_the_set == 1 else False
#consider the num of occurence of this set
set_value = int(server_term.hincrby(timestamp, dico_setname_to_redis[the_set], int(1)))

21
var/www/modules/terms/Flask_terms.py
View File

@ -330,18 +330,17 @@ def terms_management_action():
if re.match(r"[^@]+@[^@]+\.[^@]+", email):
validNotificationEmails.append(email)
# add all valid emails to the set
for email in validNotificationEmails:
r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term.lower(), email)
# enable notifications by default
r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term.lower())
# check if regex/set or simple term
#regex
if term.startswith('/') and term.endswith('/'):
r_serv_term.sadd(TrackedRegexSet_Name, term)
r_serv_term.hset(TrackedRegexDate_Name, term, today_timestamp)
# add all valid emails to the set
for email in validNotificationEmails:
r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term, email)
# enable notifications by default
r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term)
#set
elif term.startswith('\\') and term.endswith('\\'):
@ -355,11 +354,21 @@ def terms_management_action():
set_to_add = "\\" + tab_term[:-1] + ", [{}]]\\".format(match_percent)
r_serv_term.sadd(TrackedSetSet_Name, set_to_add)
r_serv_term.hset(TrackedSetDate_Name, set_to_add, today_timestamp)
# add all valid emails to the set
for email in validNotificationEmails:
r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + set_to_add, email)
# enable notifications by default
r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, set_to_add)
#simple term
else:
r_serv_term.sadd(TrackedTermsSet_Name, term.lower())
r_serv_term.hset(TrackedTermsDate_Name, term.lower(), today_timestamp)
# add all valid emails to the set
for email in validNotificationEmails:
r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term.lower(), email)
# enable notifications by default
r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term.lower())
elif action == "toggleEMailNotification":
# get the current state

7
var/www/modules/terms/templates/terms_management.html
View File

@ -129,7 +129,7 @@
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ set }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
&nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ set }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[set] %} checked {% endif %}>
</p></td>
<td style="white-space:pre">{{ notificationEMailTermMapping[term] }}</td>
<td style="white-space:pre">{{ notificationEMailTermMapping[set] }}</td>
</tr>
{% endfor %}
<!-- REGEX -->
@ -146,7 +146,7 @@
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ regex }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
&nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ regex }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[regex] %} checked {% endif %}>
</p></td>
<td style="white-space:pre">{{ notificationEMailTermMapping[term] }}</td>
<td style="white-space:pre">{{ notificationEMailTermMapping[regex] }}</td>
</tr>
{% endfor %}
<!-- Normal term -->
@ -408,3 +408,6 @@ function perform_operation(){
}
}
</script>