Implemented email notifications (bin: config.cfg additions, email sending via analyzer scripts; var: Changes to add notifications via terms management); terms_management: Fixed click handlers not being added to all tracked terms.

RegexForTermsFrequency.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python2
+# -*-coding:UTF-8 -*
+"""
+This Module is used for term frequency.
+It processes every paste coming from the global module and test the regexs
+supplied in  the term webpage.
+
+"""
+import redis
+import time
+from pubsublogger import publisher
+from packages import lib_words
+from packages import Paste
+import os
+from os import environ
+import datetime
+import calendar
+import re
+from Helper import Process
+
+# Email notifications
+from NotificationHelper import *
+
+# Config Variables
+DICO_REFRESH_TIME = 60 #s
+
+BlackListTermsSet_Name = "BlackListSetTermSet"
+TrackedTermsSet_Name = "TrackedSetTermSet"
+TrackedRegexSet_Name = "TrackedRegexSet"
+
+top_term_freq_max_set_cardinality = 20 # Max cardinality of the terms frequences set
+oneDay = 60*60*24
+top_termFreq_setName_day = ["TopTermFreq_set_day_", 1]
+top_termFreq_setName_week = ["TopTermFreq_set_week", 7]
+top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
+top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month]
+
+
+def refresh_dicos():
+    dico_regex = {} 
+    dico_regexname_to_redis = {}
+    for regex_str in server_term.smembers(TrackedRegexSet_Name):
+        dico_regex[regex_str[1:-1]] = re.compile(regex_str[1:-1])
+        dico_regexname_to_redis[regex_str[1:-1]] = regex_str
+
+    return dico_regex, dico_regexname_to_redis
+
+if __name__ == "__main__":
+    publisher.port = 6380
+    publisher.channel = "Script"
+
+    config_section = 'RegexForTermsFrequency'
+    p = Process(config_section)
+
+    # REDIS #
+    server_term = redis.StrictRedis(
+        host=p.config.get("Redis_Level_DB_TermFreq", "host"),
+        port=p.config.get("Redis_Level_DB_TermFreq", "port"),
+        db=p.config.get("Redis_Level_DB_TermFreq", "db"))
+
+    # FUNCTIONS #
+    publisher.info("RegexForTermsFrequency script started")
+
+    #compile the regex
+    dico_refresh_cooldown = time.time()
+    dico_regex, dico_regexname_to_redis = refresh_dicos()
+
+    message = p.get_from_set()
+
+    # Regex Frequency
+    while True:
+
+        if message is not None:
+            if time.time() - dico_refresh_cooldown > DICO_REFRESH_TIME:
+                dico_refresh_cooldown = time.time()
+                dico_regex, dico_regexname_to_redis = refresh_dicos()
+                print('dico got refreshed')
+
+            filename = message
+            temp = filename.split('/')
+            timestamp = calendar.timegm((int(temp[-4]), int(temp[-3]), int(temp[-2]), 0, 0, 0))
+
+            curr_set = top_termFreq_setName_day[0] + str(timestamp)
+            content = Paste.Paste(filename).get_p_content()
+
+            #iterate the word with the regex
+            for regex_str, compiled_regex in dico_regex.items():
+                matched = compiled_regex.search(content)
+
+                if matched is not None: #there is a match
+                    print('regex matched {}'.format(regex_str))
+                    matched = matched.group(0)
+                    
+                    # Add in Regex track set only if term is not in the blacklist
+                    if matched not in server_term.smembers(BlackListTermsSet_Name):
+                        
+                        # Send a notification only when the member is in the set
+                        if matched in server_term.smembers(TrackedTermsNotificationEnabled_Name):
+                            
+                            # Send to every associated email adress
+                            for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + matched):
+                                sendEmailNotification(email, matched)
+
+                        
+                        set_name = 'regex_' + dico_regexname_to_redis[regex_str]
+                        new_to_the_set = server_term.sadd(set_name, filename)
+                        new_to_the_set = True if new_to_the_set == 1 else False
+
+                        #consider the num of occurence of this term
+                        regex_value = int(server_term.hincrby(timestamp, dico_regexname_to_redis[regex_str], int(1)))
+                        #1 term per paste
+                        if new_to_the_set:
+                            regex_value_perPaste = int(server_term.hincrby("per_paste_" + str(timestamp), dico_regexname_to_redis[regex_str], int(1)))
+                            server_term.zincrby("per_paste_" + curr_set, dico_regexname_to_redis[regex_str], float(1))
+                    server_term.zincrby(curr_set, dico_regexname_to_redis[regex_str], float(1))
+                else:
+                    pass
+
+        else:
+            publisher.debug("Script RegexForTermsFrequency is Idling")
+            print "sleeping"
+            time.sleep(5)
+        message = p.get_from_set()

SetForTermsFrequency.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python2
+# -*-coding:UTF-8 -*
+"""
+This Module is used for term frequency.
+It processes every paste coming from the global module and test the sets
+supplied in  the term webpage.
+
+"""
+import redis
+import time
+from pubsublogger import publisher
+from packages import lib_words
+from packages import Paste
+import os
+import datetime
+import calendar
+import re
+import ast
+from Helper import Process
+
+# Email notifications
+from NotificationHelper import *
+
+# Config Variables
+BlackListTermsSet_Name = "BlackListSetTermSet"
+TrackedTermsSet_Name = "TrackedSetTermSet"
+TrackedRegexSet_Name = "TrackedRegexSet"
+TrackedSetSet_Name = "TrackedSetSet"
+
+top_term_freq_max_set_cardinality = 20 # Max cardinality of the terms frequences set
+oneDay = 60*60*24
+top_termFreq_setName_day = ["TopTermFreq_set_day_", 1]
+top_termFreq_setName_week = ["TopTermFreq_set_week", 7]
+top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
+top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month]
+
+def add_quote_inside_tab(tab):
+    quoted_tab = "["
+    for elem in tab[1:-1].split(','):
+        elem = elem.lstrip().strip()
+        quoted_tab += "\'{}\', ".format(elem)
+    quoted_tab = quoted_tab[:-2] #remove trailing ,
+    quoted_tab += "]"
+    return str(quoted_tab)
+
+if __name__ == "__main__":
+    publisher.port = 6380
+    publisher.channel = "Script"
+
+    config_section = 'SetForTermsFrequency'
+    p = Process(config_section)
+
+    # REDIS #
+    server_term = redis.StrictRedis(
+        host=p.config.get("Redis_Level_DB_TermFreq", "host"),
+        port=p.config.get("Redis_Level_DB_TermFreq", "port"),
+        db=p.config.get("Redis_Level_DB_TermFreq", "db"))
+
+    # FUNCTIONS #
+    publisher.info("RegexForTermsFrequency script started")
+
+    #get the dico and matching percent
+    dico_percent = {}
+    dico_set_tab = {}
+    dico_setname_to_redis = {}
+    for set_str in server_term.smembers(TrackedSetSet_Name):
+        tab_set = set_str[1:-1]
+        tab_set = add_quote_inside_tab(tab_set)
+        perc_finder = re.compile("\[[0-9]{1,3}\]").search(tab_set)
+        if perc_finder is not None:
+            match_percent = perc_finder.group(0)[1:-1]
+            dico_percent[tab_set] = float(match_percent)
+            dico_set_tab[tab_set] = ast.literal_eval(tab_set)
+            dico_setname_to_redis[tab_set] = set_str
+        else:
+            continue
+
+
+    message = p.get_from_set()
+
+    while True:
+
+        if message is not None:
+            filename = message
+            temp = filename.split('/')
+            timestamp = calendar.timegm((int(temp[-4]), int(temp[-3]), int(temp[-2]), 0, 0, 0))
+            content = Paste.Paste(filename).get_p_content()
+
+            curr_set = top_termFreq_setName_day[0] + str(timestamp)
+
+            #iterate over the words of the file
+            match_dico = {}
+            for word in content.split():
+                for cur_set, array_set in dico_set_tab.items():
+                    for w_set in array_set[:-1]: #avoid the percent matching
+                        if word == w_set:
+                            try:
+                                match_dico[str(array_set)] += 1
+                            except KeyError:
+                                match_dico[str(array_set)] = 1
+
+            #compute matching %
+            for the_set, matchingNum in match_dico.items():
+                eff_percent = float(matchingNum) / float((len(ast.literal_eval(the_set))-1)) * 100 #-1 bc if the percent matching
+                if eff_percent >= dico_percent[the_set]:
+                    
+                    # Send a notification only when the member is in the set
+                    if the_set in server_term.smembers(TrackedTermsNotificationEnabled_Name):
+                        
+                        # Send to every associated email adress
+                        for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + the_set):
+                            sendEmailNotification(email, the_set)
+                    
+                    print(the_set, "matched in", filename)
+                    set_name = 'set_' + dico_setname_to_redis[the_set]
+                    new_to_the_set = server_term.sadd(set_name, filename)
+                    new_to_the_set = True if new_to_the_set == 1 else False
+                    
+                    #consider the num of occurence of this set
+                    set_value = int(server_term.hincrby(timestamp, dico_setname_to_redis[the_set], int(1)))
+
+                    # FIXME - avoid using per paste as a set is checked over the entire paste
+                    #1 term per paste
+                    if new_to_the_set:
+                        set_value_perPaste = int(server_term.hincrby("per_paste_" + str(timestamp), dico_setname_to_redis[the_set], int(1)))
+                        server_term.zincrby("per_paste_" + curr_set, dico_setname_to_redis[the_set], float(1))
+                server_term.zincrby(curr_set, dico_setname_to_redis[the_set], float(1))
+
+
+        else:
+            publisher.debug("Script RegexForTermsFrequency is Idling")
+            print "sleeping"
+            time.sleep(5)
+        message = p.get_from_set()

bin/Curve.py

@@ -35,6 +35,9 @@ import calendar
 
 from Helper import Process
 
+# Email notifications
+from NotificationHelper import *
+
 # Config Variables
 BlackListTermsSet_Name = "BlackListSetTermSet"
 TrackedTermsSet_Name = "TrackedSetTermSet"
@@ -45,7 +48,6 @@ top_termFreq_setName_week = ["TopTermFreq_set_week", 7]
 top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
 top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month]
 
-
 def check_if_tracked_term(term, path):
     if term in server_term.smembers(TrackedTermsSet_Name):
         #add_paste to tracked_word_set
@@ -54,6 +56,13 @@ def check_if_tracked_term(term, path):
         print term, 'addded', set_name, '->', path
         p.populate_set_out("New Term added", 'CurveManageTopSets')
 
+        # Send a notification only when the member is in the set
+        if term in server_term.smembers(TrackedTermsNotificationEnabled_Name):
+            
+            # Send to every associated email adress
+            for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + term):
+                sendEmailNotification(email, term)
+
 
 def getValueOverRange(word, startDate, num_day):
     to_return = 0

bin/NotificationHelper.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python2
+# -*-coding:UTF-8 -*
+
+import ConfigParser
+import os
+import smtplib
+from email.MIMEMultipart import MIMEMultipart
+from email.mime.text import MIMEText
+
+"""
+This module allows the global configuration and management of notification settings and methods.
+"""
+
+# CONFIG #
+configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
+
+# notifications enabled/disabled
+TrackedTermsNotificationEnabled_Name = "TrackedNotifications"
+
+# associated notification email addresses for a specific term`
+# Keys will be e.g. TrackedNotificationEmails<TERMNAME>
+TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_"
+
+def sendEmailNotification(recipient, term):
+    
+    if not os.path.exists(configfile):
+        raise Exception('Unable to find the configuration file. \
+                    Did you set environment variables? \
+                    Or activate the virtualenv?')
+
+    cfg = ConfigParser.ConfigParser()
+    cfg.read(configfile)
+
+    sender = cfg.get("Notifications", "sender"),
+    sender_host = cfg.get("Notifications", "sender_host"),
+    sender_port = cfg.getint("Notifications", "sender_port"),
+    sender_pw = cfg.get("Notifications", "sender_pw"),
+    
+    if isinstance(sender, tuple):
+        sender = sender[0]
+
+    if isinstance(sender_host, tuple):
+        sender_host = sender_host[0]
+        
+    if isinstance(sender_port, tuple):
+        sender_port = sender_port[0]
+        
+    if isinstance(sender_pw, tuple):
+        sender_pw = sender_pw[0]    
+            
+    if (
+        sender is not None and
+        sender_host is not None and
+        sender_port is not None and
+        sender_pw is not None
+       ):
+        try:                    
+                                  
+            server_ssl = smtplib.SMTP_SSL(sender_host, sender_port)
+            server_ssl.ehlo()
+            server_ssl.login(sender, sender_pw)
+            
+            mime_msg = MIMEMultipart()
+            mime_msg['From'] = sender
+            mime_msg['To'] = recipient
+            mime_msg['Subject'] = "AIL Term Alert"
+            
+            body = "New occurrence for term: " + term
+            mime_msg.attach(MIMEText(body, 'plain'))
+            
+            server_ssl.sendmail(sender, recipient, mime_msg.as_string())
+            server_ssl.quit()
+            
+        except Exception as e:
+            print str(e)

bin/RegexForTermsFrequency.py

@@ -12,18 +12,22 @@ from pubsublogger import publisher
 from packages import lib_words
 from packages import Paste
 import os
+from os import environ
 import datetime
 import calendar
 import re
-
 from Helper import Process
 
+# Email notifications
+from NotificationHelper import *
+
 # Config Variables
 DICO_REFRESH_TIME = 60 #s
 
 BlackListTermsSet_Name = "BlackListSetTermSet"
 TrackedTermsSet_Name = "TrackedSetTermSet"
 TrackedRegexSet_Name = "TrackedRegexSet"
+
 top_term_freq_max_set_cardinality = 20 # Max cardinality of the terms frequences set
 oneDay = 60*60*24
 top_termFreq_setName_day = ["TopTermFreq_set_day_", 1]
@@ -86,8 +90,18 @@ if __name__ == "__main__":
                 if matched is not None: #there is a match
                     print('regex matched {}'.format(regex_str))
                     matched = matched.group(0)
+                    
                     # Add in Regex track set only if term is not in the blacklist
                     if matched not in server_term.smembers(BlackListTermsSet_Name):
+                        
+                        # Send a notification only when the member is in the set
+                        if matched in server_term.smembers(TrackedTermsNotificationEnabled_Name):
+                            
+                            # Send to every associated email adress
+                            for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + matched):
+                                sendEmailNotification(email, matched)
+
+                        
                         set_name = 'regex_' + dico_regexname_to_redis[regex_str]
                         new_to_the_set = server_term.sadd(set_name, filename)
                         new_to_the_set = True if new_to_the_set == 1 else False

bin/SetForTermsFrequency.py

@@ -16,14 +16,17 @@ import datetime
 import calendar
 import re
 import ast
-
 from Helper import Process
 
+# Email notifications
+from NotificationHelper import *
+
 # Config Variables
 BlackListTermsSet_Name = "BlackListSetTermSet"
 TrackedTermsSet_Name = "TrackedSetTermSet"
 TrackedRegexSet_Name = "TrackedRegexSet"
 TrackedSetSet_Name = "TrackedSetSet"
+
 top_term_freq_max_set_cardinality = 20 # Max cardinality of the terms frequences set
 oneDay = 60*60*24
 top_termFreq_setName_day = ["TopTermFreq_set_day_", 1]
@@ -100,12 +103,19 @@ if __name__ == "__main__":
             for the_set, matchingNum in match_dico.items():
                 eff_percent = float(matchingNum) / float((len(ast.literal_eval(the_set))-1)) * 100 #-1 bc if the percent matching
                 if eff_percent >= dico_percent[the_set]:
+                    
+                    # Send a notification only when the member is in the set
+                    if the_set in server_term.smembers(TrackedTermsNotificationEnabled_Name):
+                        
+                        # Send to every associated email adress
+                        for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + the_set):
+                            sendEmailNotification(email, the_set)
+                    
                     print(the_set, "matched in", filename)
                     set_name = 'set_' + dico_setname_to_redis[the_set]
                     new_to_the_set = server_term.sadd(set_name, filename)
                     new_to_the_set = True if new_to_the_set == 1 else False
                     
-
                     #consider the num of occurence of this set
                     set_value = int(server_term.hincrby(timestamp, dico_setname_to_redis[the_set], int(1)))
 

bin/packages/config.cfg.sample

@@ -18,6 +18,13 @@ pystemonpath = /home/pystemon/pystemon/
 
 sentiment_lexicon_file = sentiment/vader_lexicon.zip/vader_lexicon/vader_lexicon.txt
 
+##### Notifications ######
+[Notifications]
+sender = sender@example.com
+sender_host = smtp.example.com
+sender_port = 1337
+sender_pw = securepassword
+
 ##### Flask #####
 [Flask]
 #Maximum number of character to display in the toolip
@@ -28,22 +35,8 @@ max_preview_modal = 800
 default_display = 10
 #Number of minutes displayed for the number of processed pastes.
 minute_processed_paste = 10
-#Maximum line length authorized to make a diff between duplicates 
-DiffMaxLineLength = 10000
 
 #### Modules #### 
-[Categ]
-#Minimum number of match between the paste and the category file
-matchingThreshold=1
-
-[Credential]
-#Minimum length that a credential must have to be considered as such
-minimumLengthThreshold=3
-#Will be pushed as alert if the number of credentials is greater to that number
-criticalNumberToAlert=8
-#Will be considered as false positive if less that X matches from the top password list
-minTopPassList=5
-
 [Modules_Duplicates]
 #Number of month to look back
 maximum_month_range = 3
@@ -59,8 +52,8 @@ min_paste_size = 0.3
 threshold_stucked_module=600
 
 [Module_Mixer]
-#Define the configuration of the mixer, possible value: 1, 2 or 3
+#Define the configuration of the mixer, possible value: 1 or 2
-operation_mode = 3
+operation_mode = 1
 #Define the time that a paste will be considerate duplicate. in seconds (1day = 86400)
 ttl_duplicate = 86400
 
@@ -153,7 +146,7 @@ maxDuplicateToPushToMISP=10
 # e.g.: tcp://127.0.0.1:5556,tcp://127.0.0.1:5557
 [ZMQ_Global]
 #address = tcp://crf.circl.lu:5556
-address = tcp://127.0.0.1:5556,tcp://crf.circl.lu:5556
+address = tcp://127.0.0.1:5556
 channel = 102
 bind = tcp://127.0.0.1:5556
 

var/www/modules/terms/Flask_terms.py

@@ -42,6 +42,14 @@ TrackedRegexDate_Name = "TrackedRegexDate"
 TrackedSetSet_Name = "TrackedSetSet"
 TrackedSetDate_Name = "TrackedSetDate"
 
+# notifications enalbed/disabled
+# same value as in `bin/NotificationHelper.py`
+TrackedTermsNotificationEnabled_Name = "TrackedNotifications"
+
+# associated notification email addresses for a specific term`
+# same value as in `bin/NotificationHelper.py`
+# Keys will be e.g. TrackedNotificationEmails_<TERMNAME>
+TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_"
 
 '''CRED'''
 REGEX_CRED = '[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+'
@@ -138,11 +146,23 @@ def terms_management():
     today = today.replace(hour=0, minute=0, second=0, microsecond=0)
     today_timestamp = calendar.timegm(today.timetuple())
 
+    # Map tracking if notifications are enabled for a specific term
+    notificationEnabledDict = {}
+    
+    # Maps a specific term to the associated email addresses
+    notificationEMailTermMapping = {}
+
     #Regex
     trackReg_list = []
     trackReg_list_values = []
     trackReg_list_num_of_paste = []
     for tracked_regex in r_serv_term.smembers(TrackedRegexSet_Name):
+                
+        notificationEMailTermMapping[tracked_regex] = "\n".join(r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_regex))
+                
+        if tracked_regex not in notificationEnabledDict:
+            notificationEnabledDict[tracked_regex] = False
+                
         trackReg_list.append(tracked_regex)
         value_range = Term_getValueOverRange(tracked_regex, today_timestamp, [1, 7, 31], per_paste=per_paste_text)
 
@@ -154,11 +174,21 @@ def terms_management():
         value_range.append(term_date)
         trackReg_list_values.append(value_range)
 
+        if tracked_regex in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name):
+            notificationEnabledDict[tracked_regex] = True
+
     #Set
     trackSet_list = []
     trackSet_list_values = []
     trackSet_list_num_of_paste = []
     for tracked_set in r_serv_term.smembers(TrackedSetSet_Name):
+        
+        notificationEMailTermMapping[tracked_set] = "\n".join(r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_set))
+
+        
+        if tracked_set not in notificationEnabledDict:
+            notificationEnabledDict[tracked_set] = False
+        
         trackSet_list.append(tracked_set)
         value_range = Term_getValueOverRange(tracked_set, today_timestamp, [1, 7, 31], per_paste=per_paste_text)
 
@@ -170,11 +200,20 @@ def terms_management():
         value_range.append(term_date)
         trackSet_list_values.append(value_range)
 
+        if tracked_set in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name):
+            notificationEnabledDict[tracked_set] = True
+
     #Tracked terms
     track_list = []
     track_list_values = []
     track_list_num_of_paste = []
     for tracked_term in r_serv_term.smembers(TrackedTermsSet_Name):
+        
+        notificationEMailTermMapping[tracked_term] = "\n".join(r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_term))
+        
+        if tracked_term not in notificationEnabledDict:
+            notificationEnabledDict[tracked_term] = False
+        
         track_list.append(tracked_term)
         value_range = Term_getValueOverRange(tracked_term, today_timestamp, [1, 7, 31], per_paste=per_paste_text)
 
@@ -186,6 +225,8 @@ def terms_management():
         value_range.append(term_date)
         track_list_values.append(value_range)
 
+        if tracked_term in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name):
+            notificationEnabledDict[tracked_term] = True
 
     #blacklist terms
     black_list = []
@@ -199,7 +240,7 @@ def terms_management():
             track_list_values=track_list_values, track_list_num_of_paste=track_list_num_of_paste, 
             trackReg_list_values=trackReg_list_values, trackReg_list_num_of_paste=trackReg_list_num_of_paste,
             trackSet_list_values=trackSet_list_values, trackSet_list_num_of_paste=trackSet_list_num_of_paste,
-            per_paste=per_paste)
+            per_paste=per_paste, notificationEnabledDict=notificationEnabledDict, notificationEMailTermMapping=notificationEMailTermMapping)
 
 
 @terms.route("/terms_management_query_paste/")
@@ -267,11 +308,36 @@ def terms_management_action():
     section = request.args.get('section')
     action = request.args.get('action')
     term =  request.args.get('term')
+    notificationEmailsParam = request.args.get('emailAddresses')
+    
     if action is None or term is None:
         return "None"
     else:
         if section == "followTerm":
             if action == "add":
+                                
+                # Strip all whitespace
+                notificationEmailsParam = "".join(notificationEmailsParam.split())
+                
+                # Make a list of all passed email addresses
+                notificationEmails = notificationEmailsParam.split(",")
+                
+                validNotificationEmails = []
+                # check for valid email addresses
+                for email in notificationEmails:
+                    # Really basic validation:
+                    # has exactly one @ sign, and at least one . in the part after the @
+                    if re.match(r"[^@]+@[^@]+\.[^@]+", email):
+                        validNotificationEmails.append(email)
+                    
+                # add all valid emails to the set
+                for email in validNotificationEmails:
+                    r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term, email)
+                    print "added " + email + " for " + TrackedTermsNotificationEmailsPrefix_Name + term
+                    
+                # enable notifications by default
+                r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term.lower())
+                    
                 # check if regex/set or simple term
                 #regex
                 if term.startswith('/') and term.endswith('/'):
@@ -295,6 +361,16 @@ def terms_management_action():
                 else:
                     r_serv_term.sadd(TrackedTermsSet_Name, term.lower())
                     r_serv_term.hset(TrackedTermsDate_Name, term.lower(), today_timestamp)
+              
+            elif action == "toggleEMailNotification":
+                # get the current state
+                if term in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name):
+                    # remove it
+                    r_serv_term.srem(TrackedTermsNotificationEnabled_Name, term.lower())
+                else:
+                    # add it
+                    r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term.lower())
+                    
             #del action
             else:
                 if term.startswith('/') and term.endswith('/'):
@@ -308,6 +384,9 @@ def terms_management_action():
                     r_serv_term.srem(TrackedTermsSet_Name, term.lower())
                     r_serv_term.hdel(TrackedTermsDate_Name, term.lower())
 
+                # delete the associated notification emails too
+                r_serv_term.delete(TrackedTermsNotificationEmailsPrefix_Name + term)
+
         elif section == "blacklistTerm":
             if action == "add":
                 r_serv_term.sadd(BlackListTermsSet_Name, term.lower())

var/www/modules/terms/templates/terms_management.html

@@ -97,6 +97,7 @@
                             <div class="form-group input-group" style="margin-bottom: 30px;">
                                 <span class="input-group-addon"><span class="fa fa-eye"></span></span>
                                 <input id="followTermInput" class="form-control" placeholder="Term to track." type="text" style="max-width: 400px;">
+                                <input id="followTermEMailNotificationReceiversInput" class="form-control" placeholder="Notification E-Mails (comma separated)" type="text" style="max-width: 400px;">
                                 <button id="followTermBtn" class="btn btn-success btn-interaction" style="margin-left: 10px;" data-section="followTerm" data-action="add"> Add term</button>
                             </div>
 
@@ -110,6 +111,7 @@
                                         <th>Month occurence</th>
                                         <th># tracked paste</th>
                                         <th>Action</th>
+                                        <th>Notification E-Mails</th>
                                      </tr>
                                 </thead>
                                 <tbody>
@@ -122,10 +124,12 @@
                                       <td>{{ trackSet_list_values[loop.index0][1] }}</td>
                                       <td>{{ trackSet_list_values[loop.index0][2] }}</td>
                                       <td>{{ trackSet_list_num_of_paste[loop.index0] }}</td>
-                                      <td><p style="margin: 0px;">
+                                      <td><p style="margin: 0px; white-space: nowrap;">
                                           <span data-toggle="modal" data-target="#mymodal" data-term="{{ set }}" ><button class="btn-link" data-toggle="tooltip" data-placement="right" title="Show concerned paste(s)"><span class="glyphicon glyphicon-info-sign"></span></button></span>
                                           <button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ set }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
+                                          &nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ set }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[set] %} checked {% endif %}>
                                       </p></td>
+                                      <td style="white-space:pre">{{ notificationEMailTermMapping[term] }}</td>
                                   </tr>
                                 {% endfor %}
                                 <!-- REGEX -->
@@ -137,10 +141,12 @@
                                       <td>{{ trackReg_list_values[loop.index0][1] }}</td>
                                       <td>{{ trackReg_list_values[loop.index0][2] }}</td>
                                       <td>{{ trackReg_list_num_of_paste[loop.index0] }}</td>
-                                      <td><p style="margin: 0px;">
+                                      <td><p style="margin: 0px; white-space: nowrap;">
                                           <span data-toggle="modal" data-target="#mymodal" data-term="{{ regex }}" ><button class="btn-link" data-toggle="tooltip" data-placement="right" title="Show concerned paste(s)"><span class="glyphicon glyphicon-info-sign"></span></button></span>
                                           <button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ regex }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
+                                          &nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ regex }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[regex] %} checked {% endif %}>
                                       </p></td>
+                                      <td style="white-space:pre">{{ notificationEMailTermMapping[term] }}</td>
                                   </tr>
                                 {% endfor %}
                                 <!-- Normal term -->
@@ -152,10 +158,12 @@
                                       <td>{{ track_list_values[loop.index0][1] }}</td>
                                       <td>{{ track_list_values[loop.index0][2] }}</td>
                                       <td>{{ track_list_num_of_paste[loop.index0] }}</td>
-                                      <td><p style="margin: 0px;">
+                                      <td><p style="margin: 0px; white-space: nowrap;">
                                           <span data-toggle="modal" data-target="#mymodal" data-term="{{ term }}" ><button class="btn-link" data-toggle="tooltip" data-placement="right" title="Show concerned paste(s)"><span class="glyphicon glyphicon-info-sign"></span></button></span>
                                           <button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ term }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
+                                          &nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ term }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[term] %} checked {% endif %}>
                                       </p></td>
+                                      <td style="white-space:pre">{{ notificationEMailTermMapping[term] }}</td>
                                   </tr>
                                 {% endfor %}
                                 </tbody>
@@ -231,7 +239,61 @@
     var table_track;
     var table_black;
 
+function bindEventsForCurrentPage() {
+    // On click, get html content from url and update the corresponding modal
+    $("[data-toggle='modal']").unbind().on("click.openmodal", function (event) {
+        //console.log(data);
+        event.preventDefault();
+        var the_modal=$(this);
+        var url = "{{ url_for('terms.terms_management_query_paste')  }}?term=" + encodeURIComponent($(this).attr('data-term'));
+        $.getJSON(url, function (data) {
+            if (data.length != 0) {
+                var html_to_add = "";
+                html_to_add += "<table id=\"modal-table\" class=\"table table-striped\">";
+                html_to_add += "<thead>";
+                html_to_add += "<tr>";
+                html_to_add += "<th>Source</th>";
+                html_to_add += "<th>Date</th>";
+                html_to_add += "<th>Encoding</th>";
+                html_to_add += "<th>Size (Kb)</th>";
+                html_to_add += "<th># lines</th>";
+                html_to_add += "<th>Max length</th>";
+                html_to_add += "<th>Preview</th>";
+                html_to_add += "</tr>";
+                html_to_add += "</thead>";
+                html_to_add += "<tbody>";
+                for (i=0; i<data.length; i++) {
+                    curr_data = data[i];
+                    html_to_add += "<tr>";
+                    html_to_add += "<td>"+curr_data.source+"</td>";
+                    html_to_add += "<td>"+curr_data.date+"</td>";
+                    html_to_add += "<td>"+curr_data.encoding+"</td>";
+                    html_to_add += "<td>"+curr_data.size+"</td>";
+                    html_to_add += "<td>"+curr_data.lineinfo[0]+"</td>";
+                    html_to_add += "<td>"+curr_data.lineinfo[1]+"</td>";
+                    html_to_add += "<td><div class=\"row\"><button class=\"btn btn-xs btn-default\" data-toggle=\"popover\" data-placement=\"left\" data-content=\""+curr_data.content.replace(/\"/g, "\'")+"\">Preview content</button><a target=\"_blank\" href=\"{{ url_for('showsavedpastes.showsavedpaste') }}?paste="+curr_data.path+"&num=0\">  <button type=\"button\" class=\"btn btn-xs btn-info\">Show Paste</button></a></div></td>";
+
+                    html_to_add += "</tr>";
+                }
+                html_to_add += "</tbody>";
+                html_to_add += "</table>";
+                $("#mymodalbody").html(html_to_add);
+                $("[data-toggle=popover]").popover();
+                $("#button_show_plot").attr("href", "{{ url_for('terms.terms_plot_tool')}}"+"?term="+the_modal.attr('data-term') );
+                $('#modal-table').DataTable();
+            } else {
+                $("#mymodalbody").html("No paste containing this term has been received yet.");
+                $("#button_show_plot").attr("href", "{{ url_for('terms.terms_plot_tool')}}"+"?term="+the_modal.attr('data-term') );
+            }
+        });
+
+    });
+
+}
+
+
     $(document).ready(function(){
+        bindEventsForCurrentPage();
         activePage = $('h1.page-header').attr('data-page');
         $("#"+activePage).addClass("active");
         if({{ per_paste }} ==  1) {
@@ -266,55 +328,6 @@
 
         perform_binding();
 
-        // On click, get html content from url and update the corresponding modal
-        $("[data-toggle='modal']").on("click.openmodal", function (event) {
-            //console.log(data); 
-            event.preventDefault();
-            var the_modal=$(this);
-            var url = "{{ url_for('terms.terms_management_query_paste')  }}?term=" + encodeURIComponent($(this).attr('data-term'));
-            $.getJSON(url, function (data) {
-                if (data.length != 0) {
-                    var html_to_add = "";
-                    html_to_add += "<table id=\"modal-table\" class=\"table table-striped\">";
-                    html_to_add += "<thead>";
-                      html_to_add += "<tr>";
-                        html_to_add += "<th>Source</th>";
-                        html_to_add += "<th>Date</th>";
-                        html_to_add += "<th>Encoding</th>";
-                        html_to_add += "<th>Size (Kb)</th>";
-                        html_to_add += "<th># lines</th>";
-                        html_to_add += "<th>Max length</th>";
-                        html_to_add += "<th>Preview</th>";
-                      html_to_add += "</tr>";
-                    html_to_add += "</thead>";
-                    html_to_add += "<tbody>";
-                    for (i=0; i<data.length; i++) {
-                        curr_data = data[i];
-                            html_to_add += "<tr>";
-                                html_to_add += "<td>"+curr_data.source+"</td>";
-                                html_to_add += "<td>"+curr_data.date+"</td>";
-                                html_to_add += "<td>"+curr_data.encoding+"</td>";
-                                html_to_add += "<td>"+curr_data.size+"</td>";
-                                html_to_add += "<td>"+curr_data.lineinfo[0]+"</td>";
-                                html_to_add += "<td>"+curr_data.lineinfo[1]+"</td>";
-                                html_to_add += "<td><div class=\"row\"><button class=\"btn btn-xs btn-default\" data-toggle=\"popover\" data-placement=\"left\" data-content=\""+curr_data.content.replace(/\"/g, "\'")+"\">Preview content</button><a target=\"_blank\" href=\"{{ url_for('showsavedpastes.showsavedpaste') }}?paste="+curr_data.path+"&num=0\">  <button type=\"button\" class=\"btn btn-xs btn-info\">Show Paste</button></a></div></td>";
-
-                            html_to_add += "</tr>";
-                    }
-                    html_to_add += "</tbody>";
-                    html_to_add += "</table>";
-                    $("#mymodalbody").html(html_to_add);
-                    $("[data-toggle=popover]").popover();
-                    $("#button_show_plot").attr("href", "{{ url_for('terms.terms_plot_tool')}}"+"?term="+the_modal.attr('data-term') );
-	            $('#modal-table').DataTable();
-                } else {
-                    $("#mymodalbody").html("No paste containing this term has been received yet.");
-                    $("#button_show_plot").attr("href", "{{ url_for('terms.terms_plot_tool')}}"+"?term="+the_modal.attr('data-term') );
-                }
-            });
-
-       });
-
     $("#mymodal").on('hidden.bs.modal', function () {
         $("#mymodalbody").html("<p>Loading paste information...</p>");
         var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_for('static', filename='image/loading.gif') }}\" height='26' width='26' style='margin: 4px;'>";
@@ -339,10 +352,12 @@ function perform_operation(){
     var curr_action = $(this).attr('data-action');
     if (curr_action == "add") {
         var curr_term = $('#'+curr_section+'Input').val();
+        var email_addresses = $('#followTermEMailNotificationReceiversInput').val();
     } else {
         var curr_term = $(this).attr('data-content');
+        var email_addresses = "";
     }
-    var data_to_send = { section: curr_section, action:curr_action, term: curr_term};
+    var data_to_send = { section: curr_section, action: curr_action, term: curr_term, emailAddresses: email_addresses};
 
     if (curr_term != "") {
         console.log(data_to_send);
@@ -354,8 +369,11 @@ function perform_operation(){
                     if(json.action == "add") {
                         // query data
                         $.get("{{ url_for('terms.terms_management_query') }}", { term: json.term, section: json.section }, function(data2, status){
-                            var action_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"left\" title=\"Remove this term\" data-content=\"" + json.term + "\" data-section=\"followTerm\" data-action=\"delete\"><span class=\"glyphicon glyphicon-trash\"></span></button>"
+                            var delete_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"left\" title=\"Remove this term\" data-content=\"" + json.term + "\" data-section=\"followTerm\" data-action=\"delete\"><span class=\"glyphicon glyphicon-trash\"></span></button>";
-                            table_track.row.add( [ json.term, data2[3], data2[0], data2[1], data2[2], 0, action_button ] ).draw( false );
+                            var info_button = "<span data-toggle=\"modal\" data-target=\"#mymodal\" data-term=\"" + json.term + "\" ><button class=\"btn-link\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Show concerned paste(s)\"><span class=\"glyphicon glyphicon-info-sign\"></span></button></span>";
+                            var enabled_checkbox = "<input id=\"checkBoxEMailAlerts\" type=\"checkbox\" title=\"Toggle E-Mail notifications\" class=\"btn-link btn-interaction\" data-content=\"" + json.term + "\" data-section=\"followTerm\" data-action=\"toggleEMailNotification\" checked>";
+                            var action_buttons = "<p style=\"margin: 0px; white-space: nowrap;\">" + info_button + delete_button + " &nbsp; " + enabled_checkbox + "</p>";
+                            table_track.row.add( [ json.term, data2[3], data2[0], data2[1], data2[2], 0, action_buttons, $('#followTermEMailNotificationReceiversInput').val().replace(",", "\n") ] ).draw( false );
                             perform_binding();
                         });
                    } else if (json.action == "delete") {
@@ -390,6 +408,3 @@ function perform_operation(){
     }
 }
 </script>
-
-
-