Implemented email notifications (bin: config.cfg additions, email sending via analyzer scripts; var: Changes to add notifications via terms management); terms_management: Fixed click handlers not being added to all tracked terms.

pull/183/head
Philipp Schmied 2018-02-27 15:12:02 +01:00
parent 44491e837e
commit 5b1f0b0212
9 changed files with 542 additions and 90 deletions

123
RegexForTermsFrequency.py Executable file
View File

@ -0,0 +1,123 @@
#!/usr/bin/env python2
# -*-coding:UTF-8 -*
"""
This Module is used for term frequency.
It processes every paste coming from the global module and test the regexs
supplied in the term webpage.
"""
import redis
import time
from pubsublogger import publisher
from packages import lib_words
from packages import Paste
import os
from os import environ
import datetime
import calendar
import re
from Helper import Process
# Email notifications
from NotificationHelper import *
# Config Variables
DICO_REFRESH_TIME = 60 #s
BlackListTermsSet_Name = "BlackListSetTermSet"
TrackedTermsSet_Name = "TrackedSetTermSet"
TrackedRegexSet_Name = "TrackedRegexSet"
top_term_freq_max_set_cardinality = 20 # Max cardinality of the terms frequences set
oneDay = 60*60*24
top_termFreq_setName_day = ["TopTermFreq_set_day_", 1]
top_termFreq_setName_week = ["TopTermFreq_set_week", 7]
top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month]
def refresh_dicos():
dico_regex = {}
dico_regexname_to_redis = {}
for regex_str in server_term.smembers(TrackedRegexSet_Name):
dico_regex[regex_str[1:-1]] = re.compile(regex_str[1:-1])
dico_regexname_to_redis[regex_str[1:-1]] = regex_str
return dico_regex, dico_regexname_to_redis
if __name__ == "__main__":
publisher.port = 6380
publisher.channel = "Script"
config_section = 'RegexForTermsFrequency'
p = Process(config_section)
# REDIS #
server_term = redis.StrictRedis(
host=p.config.get("Redis_Level_DB_TermFreq", "host"),
port=p.config.get("Redis_Level_DB_TermFreq", "port"),
db=p.config.get("Redis_Level_DB_TermFreq", "db"))
# FUNCTIONS #
publisher.info("RegexForTermsFrequency script started")
#compile the regex
dico_refresh_cooldown = time.time()
dico_regex, dico_regexname_to_redis = refresh_dicos()
message = p.get_from_set()
# Regex Frequency
while True:
if message is not None:
if time.time() - dico_refresh_cooldown > DICO_REFRESH_TIME:
dico_refresh_cooldown = time.time()
dico_regex, dico_regexname_to_redis = refresh_dicos()
print('dico got refreshed')
filename = message
temp = filename.split('/')
timestamp = calendar.timegm((int(temp[-4]), int(temp[-3]), int(temp[-2]), 0, 0, 0))
curr_set = top_termFreq_setName_day[0] + str(timestamp)
content = Paste.Paste(filename).get_p_content()
#iterate the word with the regex
for regex_str, compiled_regex in dico_regex.items():
matched = compiled_regex.search(content)
if matched is not None: #there is a match
print('regex matched {}'.format(regex_str))
matched = matched.group(0)
# Add in Regex track set only if term is not in the blacklist
if matched not in server_term.smembers(BlackListTermsSet_Name):
# Send a notification only when the member is in the set
if matched in server_term.smembers(TrackedTermsNotificationEnabled_Name):
# Send to every associated email adress
for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + matched):
sendEmailNotification(email, matched)
set_name = 'regex_' + dico_regexname_to_redis[regex_str]
new_to_the_set = server_term.sadd(set_name, filename)
new_to_the_set = True if new_to_the_set == 1 else False
#consider the num of occurence of this term
regex_value = int(server_term.hincrby(timestamp, dico_regexname_to_redis[regex_str], int(1)))
#1 term per paste
if new_to_the_set:
regex_value_perPaste = int(server_term.hincrby("per_paste_" + str(timestamp), dico_regexname_to_redis[regex_str], int(1)))
server_term.zincrby("per_paste_" + curr_set, dico_regexname_to_redis[regex_str], float(1))
server_term.zincrby(curr_set, dico_regexname_to_redis[regex_str], float(1))
else:
pass
else:
publisher.debug("Script RegexForTermsFrequency is Idling")
print "sleeping"
time.sleep(5)
message = p.get_from_set()

134
SetForTermsFrequency.py Executable file
View File

@ -0,0 +1,134 @@
#!/usr/bin/env python2
# -*-coding:UTF-8 -*
"""
This Module is used for term frequency.
It processes every paste coming from the global module and test the sets
supplied in the term webpage.
"""
import redis
import time
from pubsublogger import publisher
from packages import lib_words
from packages import Paste
import os
import datetime
import calendar
import re
import ast
from Helper import Process
# Email notifications
from NotificationHelper import *
# Config Variables
BlackListTermsSet_Name = "BlackListSetTermSet"
TrackedTermsSet_Name = "TrackedSetTermSet"
TrackedRegexSet_Name = "TrackedRegexSet"
TrackedSetSet_Name = "TrackedSetSet"
top_term_freq_max_set_cardinality = 20 # Max cardinality of the terms frequences set
oneDay = 60*60*24
top_termFreq_setName_day = ["TopTermFreq_set_day_", 1]
top_termFreq_setName_week = ["TopTermFreq_set_week", 7]
top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month]
def add_quote_inside_tab(tab):
quoted_tab = "["
for elem in tab[1:-1].split(','):
elem = elem.lstrip().strip()
quoted_tab += "\'{}\', ".format(elem)
quoted_tab = quoted_tab[:-2] #remove trailing ,
quoted_tab += "]"
return str(quoted_tab)
if __name__ == "__main__":
publisher.port = 6380
publisher.channel = "Script"
config_section = 'SetForTermsFrequency'
p = Process(config_section)
# REDIS #
server_term = redis.StrictRedis(
host=p.config.get("Redis_Level_DB_TermFreq", "host"),
port=p.config.get("Redis_Level_DB_TermFreq", "port"),
db=p.config.get("Redis_Level_DB_TermFreq", "db"))
# FUNCTIONS #
publisher.info("RegexForTermsFrequency script started")
#get the dico and matching percent
dico_percent = {}
dico_set_tab = {}
dico_setname_to_redis = {}
for set_str in server_term.smembers(TrackedSetSet_Name):
tab_set = set_str[1:-1]
tab_set = add_quote_inside_tab(tab_set)
perc_finder = re.compile("\[[0-9]{1,3}\]").search(tab_set)
if perc_finder is not None:
match_percent = perc_finder.group(0)[1:-1]
dico_percent[tab_set] = float(match_percent)
dico_set_tab[tab_set] = ast.literal_eval(tab_set)
dico_setname_to_redis[tab_set] = set_str
else:
continue
message = p.get_from_set()
while True:
if message is not None:
filename = message
temp = filename.split('/')
timestamp = calendar.timegm((int(temp[-4]), int(temp[-3]), int(temp[-2]), 0, 0, 0))
content = Paste.Paste(filename).get_p_content()
curr_set = top_termFreq_setName_day[0] + str(timestamp)
#iterate over the words of the file
match_dico = {}
for word in content.split():
for cur_set, array_set in dico_set_tab.items():
for w_set in array_set[:-1]: #avoid the percent matching
if word == w_set:
try:
match_dico[str(array_set)] += 1
except KeyError:
match_dico[str(array_set)] = 1
#compute matching %
for the_set, matchingNum in match_dico.items():
eff_percent = float(matchingNum) / float((len(ast.literal_eval(the_set))-1)) * 100 #-1 bc if the percent matching
if eff_percent >= dico_percent[the_set]:
# Send a notification only when the member is in the set
if the_set in server_term.smembers(TrackedTermsNotificationEnabled_Name):
# Send to every associated email adress
for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + the_set):
sendEmailNotification(email, the_set)
print(the_set, "matched in", filename)
set_name = 'set_' + dico_setname_to_redis[the_set]
new_to_the_set = server_term.sadd(set_name, filename)
new_to_the_set = True if new_to_the_set == 1 else False
#consider the num of occurence of this set
set_value = int(server_term.hincrby(timestamp, dico_setname_to_redis[the_set], int(1)))
# FIXME - avoid using per paste as a set is checked over the entire paste
#1 term per paste
if new_to_the_set:
set_value_perPaste = int(server_term.hincrby("per_paste_" + str(timestamp), dico_setname_to_redis[the_set], int(1)))
server_term.zincrby("per_paste_" + curr_set, dico_setname_to_redis[the_set], float(1))
server_term.zincrby(curr_set, dico_setname_to_redis[the_set], float(1))
else:
publisher.debug("Script RegexForTermsFrequency is Idling")
print "sleeping"
time.sleep(5)
message = p.get_from_set()

View File

@ -35,6 +35,9 @@ import calendar
from Helper import Process
# Email notifications
from NotificationHelper import *
# Config Variables
BlackListTermsSet_Name = "BlackListSetTermSet"
TrackedTermsSet_Name = "TrackedSetTermSet"
@ -45,7 +48,6 @@ top_termFreq_setName_week = ["TopTermFreq_set_week", 7]
top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month]
def check_if_tracked_term(term, path):
if term in server_term.smembers(TrackedTermsSet_Name):
#add_paste to tracked_word_set
@ -54,6 +56,13 @@ def check_if_tracked_term(term, path):
print term, 'addded', set_name, '->', path
p.populate_set_out("New Term added", 'CurveManageTopSets')
# Send a notification only when the member is in the set
if term in server_term.smembers(TrackedTermsNotificationEnabled_Name):
# Send to every associated email adress
for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + term):
sendEmailNotification(email, term)
def getValueOverRange(word, startDate, num_day):
to_return = 0

75
bin/NotificationHelper.py Executable file
View File

@ -0,0 +1,75 @@
#!/usr/bin/env python2
# -*-coding:UTF-8 -*
import ConfigParser
import os
import smtplib
from email.MIMEMultipart import MIMEMultipart
from email.mime.text import MIMEText
"""
This module allows the global configuration and management of notification settings and methods.
"""
# CONFIG #
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
# notifications enabled/disabled
TrackedTermsNotificationEnabled_Name = "TrackedNotifications"
# associated notification email addresses for a specific term`
# Keys will be e.g. TrackedNotificationEmails<TERMNAME>
TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_"
def sendEmailNotification(recipient, term):
if not os.path.exists(configfile):
raise Exception('Unable to find the configuration file. \
Did you set environment variables? \
Or activate the virtualenv?')
cfg = ConfigParser.ConfigParser()
cfg.read(configfile)
sender = cfg.get("Notifications", "sender"),
sender_host = cfg.get("Notifications", "sender_host"),
sender_port = cfg.getint("Notifications", "sender_port"),
sender_pw = cfg.get("Notifications", "sender_pw"),
if isinstance(sender, tuple):
sender = sender[0]
if isinstance(sender_host, tuple):
sender_host = sender_host[0]
if isinstance(sender_port, tuple):
sender_port = sender_port[0]
if isinstance(sender_pw, tuple):
sender_pw = sender_pw[0]
if (
sender is not None and
sender_host is not None and
sender_port is not None and
sender_pw is not None
):
try:
server_ssl = smtplib.SMTP_SSL(sender_host, sender_port)
server_ssl.ehlo()
server_ssl.login(sender, sender_pw)
mime_msg = MIMEMultipart()
mime_msg['From'] = sender
mime_msg['To'] = recipient
mime_msg['Subject'] = "AIL Term Alert"
body = "New occurrence for term: " + term
mime_msg.attach(MIMEText(body, 'plain'))
server_ssl.sendmail(sender, recipient, mime_msg.as_string())
server_ssl.quit()
except Exception as e:
print str(e)

View File

@ -12,18 +12,22 @@ from pubsublogger import publisher
from packages import lib_words
from packages import Paste
import os
from os import environ
import datetime
import calendar
import re
from Helper import Process
# Email notifications
from NotificationHelper import *
# Config Variables
DICO_REFRESH_TIME = 60 #s
BlackListTermsSet_Name = "BlackListSetTermSet"
TrackedTermsSet_Name = "TrackedSetTermSet"
TrackedRegexSet_Name = "TrackedRegexSet"
top_term_freq_max_set_cardinality = 20 # Max cardinality of the terms frequences set
oneDay = 60*60*24
top_termFreq_setName_day = ["TopTermFreq_set_day_", 1]
@ -86,8 +90,18 @@ if __name__ == "__main__":
if matched is not None: #there is a match
print('regex matched {}'.format(regex_str))
matched = matched.group(0)
# Add in Regex track set only if term is not in the blacklist
if matched not in server_term.smembers(BlackListTermsSet_Name):
# Send a notification only when the member is in the set
if matched in server_term.smembers(TrackedTermsNotificationEnabled_Name):
# Send to every associated email adress
for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + matched):
sendEmailNotification(email, matched)
set_name = 'regex_' + dico_regexname_to_redis[regex_str]
new_to_the_set = server_term.sadd(set_name, filename)
new_to_the_set = True if new_to_the_set == 1 else False

View File

@ -16,14 +16,17 @@ import datetime
import calendar
import re
import ast
from Helper import Process
# Email notifications
from NotificationHelper import *
# Config Variables
BlackListTermsSet_Name = "BlackListSetTermSet"
TrackedTermsSet_Name = "TrackedSetTermSet"
TrackedRegexSet_Name = "TrackedRegexSet"
TrackedSetSet_Name = "TrackedSetSet"
top_term_freq_max_set_cardinality = 20 # Max cardinality of the terms frequences set
oneDay = 60*60*24
top_termFreq_setName_day = ["TopTermFreq_set_day_", 1]
@ -100,12 +103,19 @@ if __name__ == "__main__":
for the_set, matchingNum in match_dico.items():
eff_percent = float(matchingNum) / float((len(ast.literal_eval(the_set))-1)) * 100 #-1 bc if the percent matching
if eff_percent >= dico_percent[the_set]:
# Send a notification only when the member is in the set
if the_set in server_term.smembers(TrackedTermsNotificationEnabled_Name):
# Send to every associated email adress
for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + the_set):
sendEmailNotification(email, the_set)
print(the_set, "matched in", filename)
set_name = 'set_' + dico_setname_to_redis[the_set]
new_to_the_set = server_term.sadd(set_name, filename)
new_to_the_set = True if new_to_the_set == 1 else False
#consider the num of occurence of this set
set_value = int(server_term.hincrby(timestamp, dico_setname_to_redis[the_set], int(1)))

View File

@ -18,32 +18,25 @@ pystemonpath = /home/pystemon/pystemon/
sentiment_lexicon_file = sentiment/vader_lexicon.zip/vader_lexicon/vader_lexicon.txt
##### Notifications ######
[Notifications]
sender = sender@example.com
sender_host = smtp.example.com
sender_port = 1337
sender_pw = securepassword
##### Flask #####
[Flask]
#Maximum number of character to display in the toolip
max_preview_char = 250
max_preview_char = 250
#Maximum number of character to display in the modal
max_preview_modal = 800
max_preview_modal = 800
#Default number of header to display in trending graphs
default_display = 10
#Number of minutes displayed for the number of processed pastes.
minute_processed_paste = 10
#Maximum line length authorized to make a diff between duplicates
DiffMaxLineLength = 10000
#### Modules ####
[Categ]
#Minimum number of match between the paste and the category file
matchingThreshold=1
[Credential]
#Minimum length that a credential must have to be considered as such
minimumLengthThreshold=3
#Will be pushed as alert if the number of credentials is greater to that number
criticalNumberToAlert=8
#Will be considered as false positive if less that X matches from the top password list
minTopPassList=5
#### Modules ####
[Modules_Duplicates]
#Number of month to look back
maximum_month_range = 3
@ -59,8 +52,8 @@ min_paste_size = 0.3
threshold_stucked_module=600
[Module_Mixer]
#Define the configuration of the mixer, possible value: 1, 2 or 3
operation_mode = 3
#Define the configuration of the mixer, possible value: 1 or 2
operation_mode = 1
#Define the time that a paste will be considerate duplicate. in seconds (1day = 86400)
ttl_duplicate = 86400
@ -153,7 +146,7 @@ maxDuplicateToPushToMISP=10
# e.g.: tcp://127.0.0.1:5556,tcp://127.0.0.1:5557
[ZMQ_Global]
#address = tcp://crf.circl.lu:5556
address = tcp://127.0.0.1:5556,tcp://crf.circl.lu:5556
address = tcp://127.0.0.1:5556
channel = 102
bind = tcp://127.0.0.1:5556

View File

@ -42,6 +42,14 @@ TrackedRegexDate_Name = "TrackedRegexDate"
TrackedSetSet_Name = "TrackedSetSet"
TrackedSetDate_Name = "TrackedSetDate"
# notifications enalbed/disabled
# same value as in `bin/NotificationHelper.py`
TrackedTermsNotificationEnabled_Name = "TrackedNotifications"
# associated notification email addresses for a specific term`
# same value as in `bin/NotificationHelper.py`
# Keys will be e.g. TrackedNotificationEmails_<TERMNAME>
TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_"
'''CRED'''
REGEX_CRED = '[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+'
@ -138,11 +146,23 @@ def terms_management():
today = today.replace(hour=0, minute=0, second=0, microsecond=0)
today_timestamp = calendar.timegm(today.timetuple())
# Map tracking if notifications are enabled for a specific term
notificationEnabledDict = {}
# Maps a specific term to the associated email addresses
notificationEMailTermMapping = {}
#Regex
trackReg_list = []
trackReg_list_values = []
trackReg_list_num_of_paste = []
for tracked_regex in r_serv_term.smembers(TrackedRegexSet_Name):
notificationEMailTermMapping[tracked_regex] = "\n".join(r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_regex))
if tracked_regex not in notificationEnabledDict:
notificationEnabledDict[tracked_regex] = False
trackReg_list.append(tracked_regex)
value_range = Term_getValueOverRange(tracked_regex, today_timestamp, [1, 7, 31], per_paste=per_paste_text)
@ -154,11 +174,21 @@ def terms_management():
value_range.append(term_date)
trackReg_list_values.append(value_range)
if tracked_regex in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name):
notificationEnabledDict[tracked_regex] = True
#Set
trackSet_list = []
trackSet_list_values = []
trackSet_list_num_of_paste = []
for tracked_set in r_serv_term.smembers(TrackedSetSet_Name):
notificationEMailTermMapping[tracked_set] = "\n".join(r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_set))
if tracked_set not in notificationEnabledDict:
notificationEnabledDict[tracked_set] = False
trackSet_list.append(tracked_set)
value_range = Term_getValueOverRange(tracked_set, today_timestamp, [1, 7, 31], per_paste=per_paste_text)
@ -170,11 +200,20 @@ def terms_management():
value_range.append(term_date)
trackSet_list_values.append(value_range)
if tracked_set in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name):
notificationEnabledDict[tracked_set] = True
#Tracked terms
track_list = []
track_list_values = []
track_list_num_of_paste = []
for tracked_term in r_serv_term.smembers(TrackedTermsSet_Name):
notificationEMailTermMapping[tracked_term] = "\n".join(r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_term))
if tracked_term not in notificationEnabledDict:
notificationEnabledDict[tracked_term] = False
track_list.append(tracked_term)
value_range = Term_getValueOverRange(tracked_term, today_timestamp, [1, 7, 31], per_paste=per_paste_text)
@ -186,6 +225,8 @@ def terms_management():
value_range.append(term_date)
track_list_values.append(value_range)
if tracked_term in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name):
notificationEnabledDict[tracked_term] = True
#blacklist terms
black_list = []
@ -199,7 +240,7 @@ def terms_management():
track_list_values=track_list_values, track_list_num_of_paste=track_list_num_of_paste,
trackReg_list_values=trackReg_list_values, trackReg_list_num_of_paste=trackReg_list_num_of_paste,
trackSet_list_values=trackSet_list_values, trackSet_list_num_of_paste=trackSet_list_num_of_paste,
per_paste=per_paste)
per_paste=per_paste, notificationEnabledDict=notificationEnabledDict, notificationEMailTermMapping=notificationEMailTermMapping)
@terms.route("/terms_management_query_paste/")
@ -267,11 +308,36 @@ def terms_management_action():
section = request.args.get('section')
action = request.args.get('action')
term = request.args.get('term')
notificationEmailsParam = request.args.get('emailAddresses')
if action is None or term is None:
return "None"
else:
if section == "followTerm":
if action == "add":
# Strip all whitespace
notificationEmailsParam = "".join(notificationEmailsParam.split())
# Make a list of all passed email addresses
notificationEmails = notificationEmailsParam.split(",")
validNotificationEmails = []
# check for valid email addresses
for email in notificationEmails:
# Really basic validation:
# has exactly one @ sign, and at least one . in the part after the @
if re.match(r"[^@]+@[^@]+\.[^@]+", email):
validNotificationEmails.append(email)
# add all valid emails to the set
for email in validNotificationEmails:
r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term, email)
print "added " + email + " for " + TrackedTermsNotificationEmailsPrefix_Name + term
# enable notifications by default
r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term.lower())
# check if regex/set or simple term
#regex
if term.startswith('/') and term.endswith('/'):
@ -295,6 +361,16 @@ def terms_management_action():
else:
r_serv_term.sadd(TrackedTermsSet_Name, term.lower())
r_serv_term.hset(TrackedTermsDate_Name, term.lower(), today_timestamp)
elif action == "toggleEMailNotification":
# get the current state
if term in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name):
# remove it
r_serv_term.srem(TrackedTermsNotificationEnabled_Name, term.lower())
else:
# add it
r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term.lower())
#del action
else:
if term.startswith('/') and term.endswith('/'):
@ -308,6 +384,9 @@ def terms_management_action():
r_serv_term.srem(TrackedTermsSet_Name, term.lower())
r_serv_term.hdel(TrackedTermsDate_Name, term.lower())
# delete the associated notification emails too
r_serv_term.delete(TrackedTermsNotificationEmailsPrefix_Name + term)
elif section == "blacklistTerm":
if action == "add":
r_serv_term.sadd(BlackListTermsSet_Name, term.lower())

View File

@ -42,7 +42,7 @@
<!-- Modal -->
<div id="mymodal" class="modal fade" role="dialog">
<div class="modal-dialog modal-lg">
<!-- Modal content-->
<div id="mymodalcontent" class="modal-content">
<div id="mymodalbody" class="modal-body" max-width="8500px">
@ -87,7 +87,7 @@
</div>
<div class="panel-body">
<div style="margin-bottom: 10px;">
<div style="margin-bottom: 10px;">
<table>
<tr><td><b>Regex</b>: surround the term by '<b>/</b>'. </td> <td><b style="margin-left: 20px;">/([a-z])\w+([a-z])\n/</b></td></tr>
<tr><td><b>Set of terms</b>: surround the list by '<b>\</b>'. </td> <td><b style="margin-left: 20px;">\[term1, term2, ...]\</b></td></tr>
@ -97,6 +97,7 @@
<div class="form-group input-group" style="margin-bottom: 30px;">
<span class="input-group-addon"><span class="fa fa-eye"></span></span>
<input id="followTermInput" class="form-control" placeholder="Term to track." type="text" style="max-width: 400px;">
<input id="followTermEMailNotificationReceiversInput" class="form-control" placeholder="Notification E-Mails (comma separated)" type="text" style="max-width: 400px;">
<button id="followTermBtn" class="btn btn-success btn-interaction" style="margin-left: 10px;" data-section="followTerm" data-action="add"> Add term</button>
</div>
@ -110,6 +111,7 @@
<th>Month occurence</th>
<th># tracked paste</th>
<th>Action</th>
<th>Notification E-Mails</th>
</tr>
</thead>
<tbody>
@ -122,10 +124,12 @@
<td>{{ trackSet_list_values[loop.index0][1] }}</td>
<td>{{ trackSet_list_values[loop.index0][2] }}</td>
<td>{{ trackSet_list_num_of_paste[loop.index0] }}</td>
<td><p style="margin: 0px;">
<td><p style="margin: 0px; white-space: nowrap;">
<span data-toggle="modal" data-target="#mymodal" data-term="{{ set }}" ><button class="btn-link" data-toggle="tooltip" data-placement="right" title="Show concerned paste(s)"><span class="glyphicon glyphicon-info-sign"></span></button></span>
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ set }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
&nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ set }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[set] %} checked {% endif %}>
</p></td>
<td style="white-space:pre">{{ notificationEMailTermMapping[term] }}</td>
</tr>
{% endfor %}
<!-- REGEX -->
@ -137,10 +141,12 @@
<td>{{ trackReg_list_values[loop.index0][1] }}</td>
<td>{{ trackReg_list_values[loop.index0][2] }}</td>
<td>{{ trackReg_list_num_of_paste[loop.index0] }}</td>
<td><p style="margin: 0px;">
<td><p style="margin: 0px; white-space: nowrap;">
<span data-toggle="modal" data-target="#mymodal" data-term="{{ regex }}" ><button class="btn-link" data-toggle="tooltip" data-placement="right" title="Show concerned paste(s)"><span class="glyphicon glyphicon-info-sign"></span></button></span>
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ regex }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
&nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ regex }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[regex] %} checked {% endif %}>
</p></td>
<td style="white-space:pre">{{ notificationEMailTermMapping[term] }}</td>
</tr>
{% endfor %}
<!-- Normal term -->
@ -152,10 +158,12 @@
<td>{{ track_list_values[loop.index0][1] }}</td>
<td>{{ track_list_values[loop.index0][2] }}</td>
<td>{{ track_list_num_of_paste[loop.index0] }}</td>
<td><p style="margin: 0px;">
<td><p style="margin: 0px; white-space: nowrap;">
<span data-toggle="modal" data-target="#mymodal" data-term="{{ term }}" ><button class="btn-link" data-toggle="tooltip" data-placement="right" title="Show concerned paste(s)"><span class="glyphicon glyphicon-info-sign"></span></button></span>
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ term }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
&nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ term }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[term] %} checked {% endif %}>
</p></td>
<td style="white-space:pre">{{ notificationEMailTermMapping[term] }}</td>
</tr>
{% endfor %}
</tbody>
@ -168,7 +176,7 @@
<!-- /.panel -->
</div>
</div>
<!-- Panel OPTIONS -->
<div class="col-lg-12">
<div class="row">
@ -212,7 +220,7 @@
</div>
<!-- /.panel -->
</div>
</div>
</div>
<!-- /.row -->
</div>
@ -231,7 +239,61 @@
var table_track;
var table_black;
function bindEventsForCurrentPage() {
// On click, get html content from url and update the corresponding modal
$("[data-toggle='modal']").unbind().on("click.openmodal", function (event) {
//console.log(data);
event.preventDefault();
var the_modal=$(this);
var url = "{{ url_for('terms.terms_management_query_paste') }}?term=" + encodeURIComponent($(this).attr('data-term'));
$.getJSON(url, function (data) {
if (data.length != 0) {
var html_to_add = "";
html_to_add += "<table id=\"modal-table\" class=\"table table-striped\">";
html_to_add += "<thead>";
html_to_add += "<tr>";
html_to_add += "<th>Source</th>";
html_to_add += "<th>Date</th>";
html_to_add += "<th>Encoding</th>";
html_to_add += "<th>Size (Kb)</th>";
html_to_add += "<th># lines</th>";
html_to_add += "<th>Max length</th>";
html_to_add += "<th>Preview</th>";
html_to_add += "</tr>";
html_to_add += "</thead>";
html_to_add += "<tbody>";
for (i=0; i<data.length; i++) {
curr_data = data[i];
html_to_add += "<tr>";
html_to_add += "<td>"+curr_data.source+"</td>";
html_to_add += "<td>"+curr_data.date+"</td>";
html_to_add += "<td>"+curr_data.encoding+"</td>";
html_to_add += "<td>"+curr_data.size+"</td>";
html_to_add += "<td>"+curr_data.lineinfo[0]+"</td>";
html_to_add += "<td>"+curr_data.lineinfo[1]+"</td>";
html_to_add += "<td><div class=\"row\"><button class=\"btn btn-xs btn-default\" data-toggle=\"popover\" data-placement=\"left\" data-content=\""+curr_data.content.replace(/\"/g, "\'")+"\">Preview content</button><a target=\"_blank\" href=\"{{ url_for('showsavedpastes.showsavedpaste') }}?paste="+curr_data.path+"&num=0\"> <button type=\"button\" class=\"btn btn-xs btn-info\">Show Paste</button></a></div></td>";
html_to_add += "</tr>";
}
html_to_add += "</tbody>";
html_to_add += "</table>";
$("#mymodalbody").html(html_to_add);
$("[data-toggle=popover]").popover();
$("#button_show_plot").attr("href", "{{ url_for('terms.terms_plot_tool')}}"+"?term="+the_modal.attr('data-term') );
$('#modal-table').DataTable();
} else {
$("#mymodalbody").html("No paste containing this term has been received yet.");
$("#button_show_plot").attr("href", "{{ url_for('terms.terms_plot_tool')}}"+"?term="+the_modal.attr('data-term') );
}
});
});
}
$(document).ready(function(){
bindEventsForCurrentPage();
activePage = $('h1.page-header').attr('data-page');
$("#"+activePage).addClass("active");
if({{ per_paste }} == 1) {
@ -256,7 +318,7 @@
$("#followTermInput").val("");
}
});
$("#blacklistTermInput").keyup(function(event){
if(event.keyCode == 13){
$("#blacklistTermBtn").click();
@ -266,55 +328,6 @@
perform_binding();
// On click, get html content from url and update the corresponding modal
$("[data-toggle='modal']").on("click.openmodal", function (event) {
//console.log(data);
event.preventDefault();
var the_modal=$(this);
var url = "{{ url_for('terms.terms_management_query_paste') }}?term=" + encodeURIComponent($(this).attr('data-term'));
$.getJSON(url, function (data) {
if (data.length != 0) {
var html_to_add = "";
html_to_add += "<table id=\"modal-table\" class=\"table table-striped\">";
html_to_add += "<thead>";
html_to_add += "<tr>";
html_to_add += "<th>Source</th>";
html_to_add += "<th>Date</th>";
html_to_add += "<th>Encoding</th>";
html_to_add += "<th>Size (Kb)</th>";
html_to_add += "<th># lines</th>";
html_to_add += "<th>Max length</th>";
html_to_add += "<th>Preview</th>";
html_to_add += "</tr>";
html_to_add += "</thead>";
html_to_add += "<tbody>";
for (i=0; i<data.length; i++) {
curr_data = data[i];
html_to_add += "<tr>";
html_to_add += "<td>"+curr_data.source+"</td>";
html_to_add += "<td>"+curr_data.date+"</td>";
html_to_add += "<td>"+curr_data.encoding+"</td>";
html_to_add += "<td>"+curr_data.size+"</td>";
html_to_add += "<td>"+curr_data.lineinfo[0]+"</td>";
html_to_add += "<td>"+curr_data.lineinfo[1]+"</td>";
html_to_add += "<td><div class=\"row\"><button class=\"btn btn-xs btn-default\" data-toggle=\"popover\" data-placement=\"left\" data-content=\""+curr_data.content.replace(/\"/g, "\'")+"\">Preview content</button><a target=\"_blank\" href=\"{{ url_for('showsavedpastes.showsavedpaste') }}?paste="+curr_data.path+"&num=0\"> <button type=\"button\" class=\"btn btn-xs btn-info\">Show Paste</button></a></div></td>";
html_to_add += "</tr>";
}
html_to_add += "</tbody>";
html_to_add += "</table>";
$("#mymodalbody").html(html_to_add);
$("[data-toggle=popover]").popover();
$("#button_show_plot").attr("href", "{{ url_for('terms.terms_plot_tool')}}"+"?term="+the_modal.attr('data-term') );
$('#modal-table').DataTable();
} else {
$("#mymodalbody").html("No paste containing this term has been received yet.");
$("#button_show_plot").attr("href", "{{ url_for('terms.terms_plot_tool')}}"+"?term="+the_modal.attr('data-term') );
}
});
});
$("#mymodal").on('hidden.bs.modal', function () {
$("#mymodalbody").html("<p>Loading paste information...</p>");
var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_for('static', filename='image/loading.gif') }}\" height='26' width='26' style='margin: 4px;'>";
@ -336,13 +349,15 @@ function perform_binding() {
function perform_operation(){
var curr_section = $(this).attr('data-section');
var curr_action = $(this).attr('data-action');
var curr_action = $(this).attr('data-action');
if (curr_action == "add") {
var curr_term = $('#'+curr_section+'Input').val();
var email_addresses = $('#followTermEMailNotificationReceiversInput').val();
} else {
var curr_term = $(this).attr('data-content');
var email_addresses = "";
}
var data_to_send = { section: curr_section, action:curr_action, term: curr_term};
var data_to_send = { section: curr_section, action: curr_action, term: curr_term, emailAddresses: email_addresses};
if (curr_term != "") {
console.log(data_to_send);
@ -354,8 +369,11 @@ function perform_operation(){
if(json.action == "add") {
// query data
$.get("{{ url_for('terms.terms_management_query') }}", { term: json.term, section: json.section }, function(data2, status){
var action_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"left\" title=\"Remove this term\" data-content=\"" + json.term + "\" data-section=\"followTerm\" data-action=\"delete\"><span class=\"glyphicon glyphicon-trash\"></span></button>"
table_track.row.add( [ json.term, data2[3], data2[0], data2[1], data2[2], 0, action_button ] ).draw( false );
var delete_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"left\" title=\"Remove this term\" data-content=\"" + json.term + "\" data-section=\"followTerm\" data-action=\"delete\"><span class=\"glyphicon glyphicon-trash\"></span></button>";
var info_button = "<span data-toggle=\"modal\" data-target=\"#mymodal\" data-term=\"" + json.term + "\" ><button class=\"btn-link\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Show concerned paste(s)\"><span class=\"glyphicon glyphicon-info-sign\"></span></button></span>";
var enabled_checkbox = "<input id=\"checkBoxEMailAlerts\" type=\"checkbox\" title=\"Toggle E-Mail notifications\" class=\"btn-link btn-interaction\" data-content=\"" + json.term + "\" data-section=\"followTerm\" data-action=\"toggleEMailNotification\" checked>";
var action_buttons = "<p style=\"margin: 0px; white-space: nowrap;\">" + info_button + delete_button + " &nbsp; " + enabled_checkbox + "</p>";
table_track.row.add( [ json.term, data2[3], data2[0], data2[1], data2[2], 0, action_buttons, $('#followTermEMailNotificationReceiversInput').val().replace(",", "\n") ] ).draw( false );
perform_binding();
});
} else if (json.action == "delete") {
@ -383,13 +401,10 @@ function perform_operation(){
}
}
}
});
}
}
</script>