Merge branch 'master' of github.com:CIRCL/AIL-framework

Conflicts:
	bin/launch_scripts.sh

README.md

@@ -104,6 +104,7 @@ LICENSE
     Copyright (C) 2014-2016 CIRCL - Computer Incident Response Center Luxembourg (c/o smile, security made in Lëtzebuerg, Groupement d'Intérêt Economique)
     Copyright (c) 2014-2016 Raphaël Vinot
     Copyright (c) 2014-2016 Alexandre Dulaunoy
+    Copyright (c) 2016 Sami Mokaddem
 
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU Affero General Public License as published by

bin/CreditCard.py

@@ -66,13 +66,12 @@ if __name__ == "__main__":
                     publisher.warning('{}Checked {} valid number(s)'.format(
                         to_print, len(creditcard_set)))
                     #Send to duplicate
-                    p.populate_set_out(filepath, 'Redis_Duplicate')
+                    p.populate_set_out(filepath, 'Duplicate')
                     #send to Browse_warning_paste
                     p.populate_set_out('creditcard;{}'.format(filename), 'BrowseWarningPaste')
                 else:
                     publisher.info('{}CreditCard related'.format(to_print))
         else:
             publisher.debug("Script creditcard is idling 1m")
-            print 'Sleeping'
             time.sleep(10)
 

bin/Curve.py

@@ -1,9 +1,6 @@
 #!/usr/bin/env python2
 # -*-coding:UTF-8 -*
 """
-The ZMQ_Sub_Curve Module
-============================
-
 This module is consuming the Redis-list created by the ZMQ_Sub_Curve_Q Module.
 
 This modules update a .csv file used to draw curves representing selected
@@ -14,6 +11,12 @@ words and their occurency per day.
 ..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put
 the same Subscriber name in both of them.
 
+
+This Module is also used for term frequency.
+
+/!\ Top set management is done in the module Curve_manage_top_set
+
+
 Requirements
 ------------
 
@@ -28,9 +31,39 @@ from pubsublogger import publisher
 from packages import lib_words
 import os
 import datetime
+import calendar
 
 from Helper import Process
 
+# Config Variables
+BlackListTermsSet_Name = "BlackListSetTermSet"
+TrackedTermsSet_Name = "TrackedSetTermSet"
+top_term_freq_max_set_cardinality = 20 # Max cardinality of the terms frequences set
+oneDay = 60*60*24
+top_termFreq_setName_day = ["TopTermFreq_set_day_", 1]
+top_termFreq_setName_week = ["TopTermFreq_set_week", 7]
+top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
+top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month]
+
+
+def check_if_tracked_term(term, path):
+    if term in server_term.smembers(TrackedTermsSet_Name):
+        #add_paste to tracked_word_set
+        set_name = "tracked_" + term
+        server_term.sadd(set_name, path)
+        print term, 'addded', set_name, '->', path
+        p.populate_set_out("New Term added", 'CurveManageTopSets')
+
+
+def getValueOverRange(word, startDate, num_day):
+    to_return = 0
+    for timestamp in range(startDate, startDate - num_day*oneDay, -oneDay):
+        value = server_term.hget(timestamp, word)
+        to_return += int(value) if value is not None else 0
+    return to_return
+
+
+
 if __name__ == "__main__":
     publisher.port = 6380
     publisher.channel = "Script"
@@ -44,6 +77,11 @@ if __name__ == "__main__":
         port=p.config.get("Redis_Level_DB_Curve", "port"),
         db=p.config.get("Redis_Level_DB_Curve", "db"))
 
+    server_term = redis.StrictRedis(
+        host=p.config.get("Redis_Level_DB_TermFreq", "host"),
+        port=p.config.get("Redis_Level_DB_TermFreq", "port"),
+        db=p.config.get("Redis_Level_DB_TermFreq", "db"))
+
     # FUNCTIONS #
     publisher.info("Script Curve started")
 
@@ -56,22 +94,40 @@ if __name__ == "__main__":
     message = p.get_from_set()
     prec_filename = None
     generate_new_graph = False
+
+    # Term Frequency
+    top_termFreq_setName_day = ["TopTermFreq_set_day_", 1]
+    top_termFreq_setName_week = ["TopTermFreq_set_week", 7]
+    top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
+
     while True:
+
         if message is not None:
             generate_new_graph = True
 
             filename, word, score = message.split()
             temp = filename.split('/')
             date = temp[-4] + temp[-3] + temp[-2]
+            timestamp = calendar.timegm((int(temp[-4]), int(temp[-3]), int(temp[-2]), 0, 0, 0))
+            curr_set = top_termFreq_setName_day[0] + str(timestamp)
+
 
             low_word = word.lower()
-            prev_score = r_serv1.hget(low_word, date)
-            if prev_score is not None:
-                r_serv1.hset(low_word, date, int(prev_score) + int(score))
-            else:
-                r_serv1.hset(low_word, date, score)
+            #Old curve with words in file
+            r_serv1.hincrby(low_word, date, int(score))
+
+            # Update redis
+            curr_word_value = int(server_term.hincrby(timestamp, low_word, int(score)))
+
+            # Add in set only if term is not in the blacklist
+            if low_word not in server_term.smembers(BlackListTermsSet_Name):
+                server_term.zincrby(curr_set, low_word, float(score))
+            
+            #Add more info for tracked terms
+            check_if_tracked_term(low_word, filename)
 
         else:
+
             if generate_new_graph:
                 generate_new_graph = False
                 print 'Building graph'

bin/Curve_manage_top_sets.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python2
+# -*-coding:UTF-8 -*
+"""
+
+This module manage top sets for terms frequency.
+Every 'refresh_rate' update the weekly and monthly set
+
+
+Requirements
+------------
+
+*Need running Redis instances. (Redis)
+*Categories files of words in /files/ need to be created
+*Need the ZMQ_PubSub_Tokenize_Q Module running to be able to work properly.
+
+"""
+
+import redis
+import time
+import copy
+from pubsublogger import publisher
+from packages import lib_words
+import datetime
+import calendar
+
+from Helper import Process
+
+# Config Variables
+Refresh_rate = 60*5 #sec
+BlackListTermsSet_Name = "BlackListSetTermSet"
+TrackedTermsSet_Name = "TrackedSetTermSet"
+top_term_freq_max_set_cardinality = 20 # Max cardinality of the terms frequences set
+oneDay = 60*60*24
+num_day_month = 31
+num_day_week = 7
+
+top_termFreq_setName_day = ["TopTermFreq_set_day_", 1]
+top_termFreq_setName_week = ["TopTermFreq_set_week", 7]
+top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
+top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month]
+
+
+def manage_top_set():
+    startDate = datetime.datetime.now()
+    startDate = startDate.replace(hour=0, minute=0, second=0, microsecond=0)
+    startDate = calendar.timegm(startDate.timetuple())
+
+    dico = {}
+
+    # Retreive top data (2*max_card) from days sets
+    for timestamp in range(startDate, startDate - top_termFreq_setName_month[1]*oneDay, -oneDay):
+        curr_set = top_termFreq_setName_day[0] + str(timestamp)
+        array_top_day = server_term.zrevrangebyscore(curr_set, '+inf', '-inf', withscores=True, start=0, num=top_term_freq_max_set_cardinality*2)
+
+        for word, value in array_top_day:
+            if word not in server_term.smembers(BlackListTermsSet_Name):
+                if word in dico.keys():
+                    dico[word] += value
+                else:
+                    dico[word] = value
+
+        if timestamp == startDate - num_day_week*oneDay:
+            dico_week = copy.deepcopy(dico)
+
+    # convert dico into sorted array
+    array_month = []
+    for w, v in dico.iteritems():
+        array_month.append((w, v))
+    array_month.sort(key=lambda tup: -tup[1])
+    array_month = array_month[0:20]
+
+    array_week = []
+    for w, v in dico_week.iteritems():
+        array_week.append((w, v))
+    array_week.sort(key=lambda tup: -tup[1])
+    array_week = array_week[0:20]
+
+    # suppress every terms in top sets
+    for curr_set, curr_num_day in top_termFreq_set_array[1:3]:
+        for w in server_term.zrange(curr_set, 0, -1):
+            server_term.zrem(curr_set, w)
+
+    # Add top term from sorted array in their respective sorted sets
+    for elem in array_week:
+        server_term.zadd(top_termFreq_setName_week[0], float(elem[1]), elem[0])
+  
+    for elem in array_month:
+        server_term.zadd(top_termFreq_setName_month[0], float(elem[1]), elem[0])
+
+
+
+if __name__ == '__main__':
+    # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
+    # Port of the redis instance used by pubsublogger
+    publisher.port = 6380
+    # Script is the default channel used for the modules.
+    publisher.channel = 'Script'
+
+    config_section = 'CurveManageTopSets'
+    p = Process(config_section)
+
+    server_term = redis.StrictRedis(
+        host=p.config.get("Redis_Level_DB_TermFreq", "host"),
+        port=p.config.get("Redis_Level_DB_TermFreq", "port"),
+        db=p.config.get("Redis_Level_DB_TermFreq", "db"))
+
+    publisher.info("Script Curve_manage_top_set started")
+
+    # Sent to the logging a description of the module
+    publisher.info("Manage the top sets with the data created by the module curve.")
+
+    manage_top_set()
+
+    while True:
+        # Get one message from the input queue (module only work if linked with a queue)
+        message = p.get_from_set()
+        if message is None:
+            publisher.debug("{} queue is empty, waiting".format(config_section))
+            print 'sleeping'
+            time.sleep(Refresh_rate) # sleep a long time then manage the set
+            manage_top_set()
+            continue
+

bin/LAUNCH.sh

@@ -58,6 +58,10 @@ function launching_redis {
     screen -S "Redis" -X screen -t "6380" bash -c 'redis-server '$conf_dir'6380.conf ; read x'
     sleep 0.1
     screen -S "Redis" -X screen -t "6381" bash -c 'redis-server '$conf_dir'6381.conf ; read x'
+
+    # For Words and curves
+    sleep 0.1
+    screen -S "Redis" -X screen -t "6382" bash -c 'redis-server '$conf_dir'6382.conf ; read x'
 }
 
 function launching_lvldb {
@@ -134,6 +138,8 @@ function launching_scripts {
     sleep 0.1
     screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x'
     sleep 0.1
+    screen -S "Script" -X screen -t "Curve_topsets_manager" bash -c './Curve_manage_top_sets.py; read x'
+    sleep 0.1
     screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x'
     sleep 0.1
     screen -S "Script" -X screen -t "Keys" bash -c './Keys.py; read x'
@@ -151,6 +157,9 @@ function launching_scripts {
     screen -S "Script" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x'
     sleep 0.1
     screen -S "Script" -X screen -t "Browse_warning_paste" bash -c './Browse_warning_paste.py; read x'
+    sleep 0.1
+    screen -S "Script" -X screen -t "SentimentAnalyser" bash -c './SentimentAnalyser.py; read x'
+
 }
 
 #If no params, display the help

bin/ModuleStats.py

@@ -1,7 +1,8 @@
 #!/usr/bin/env python2
 # -*-coding:UTF-8 -*
 """
-    Template for new modules
+    This module makes statistics for some modules and providers
+
 """
 
 import time
@@ -15,7 +16,7 @@ from Helper import Process
 from packages import Paste
 
 # Config Var
-max_set_cardinality = 7
+max_set_cardinality = 8
 
 def get_date_range(num_day):
     curr_date = datetime.date.today()
@@ -30,14 +31,10 @@ def get_date_range(num_day):
 def compute_most_posted(server, message):
     module, num, keyword, paste_date = message.split(';')
 
-    redis_progression_name_set = 'top_'+ module +'_set'
+    redis_progression_name_set = 'top_'+ module +'_set_' + paste_date
 
     # Add/Update in Redis
-    prev_score = server.hget(paste_date, module+'-'+keyword)
-    if prev_score is not None:
-        ok = server.hset(paste_date, module+'-'+keyword, int(prev_score) + int(num))
-    else:
-        ok = server.hset(paste_date, module+'-'+keyword, int(num))
+    server.hincrby(paste_date, module+'-'+keyword, int(num))
 
     # Compute Most Posted
     date = get_date_range(0)[0]
@@ -47,103 +44,76 @@ def compute_most_posted(server, message):
     curr_value = server.hget(date, module+'-'+keyword)
     keyword_total_sum += int(curr_value) if curr_value is not None else 0
         
-    if keyword in server.smembers(redis_progression_name_set): # if it is already in the set
-        return
+    if server.zcard(redis_progression_name_set) < max_set_cardinality:
+        server.zadd(redis_progression_name_set, float(keyword_total_sum), keyword)
 
-    if (server.scard(redis_progression_name_set) < max_set_cardinality):
-        server.sadd(redis_progression_name_set, keyword)
-
-    else: #not in the set
-        #Check value for all members
-        member_set = []
-        for keyw in server.smembers(redis_progression_name_set):
-            keyw_value = server.hget(paste_date, module+'-'+keyw)
-            if keyw_value is not None:
-                member_set.append((keyw, int(keyw_value)))
-            else: #No data for this set for today
-                member_set.append((keyw, int(0)))
-        member_set.sort(key=lambda tup: tup[1])
-        if len(member_set) > 0:
-            if member_set[0][1] < keyword_total_sum:
-                #remove min from set and add the new one
-                print module + ': adding ' +keyword+ '(' +str(keyword_total_sum)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')'
-                server.srem(redis_progression_name_set, member_set[0][0])
-                server.sadd(redis_progression_name_set, keyword)
+    else: # not in set
+        member_set = server.zrangebyscore(redis_progression_name_set, '-inf', '+inf', withscores=True, start=0, num=1)
+        # Member set is a list of (value, score) pairs
+        if int(member_set[0][1]) < keyword_total_sum:
+            #remove min from set and add the new one
+            print module + ': adding ' +keyword+ '(' +str(keyword_total_sum)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')'
+            server.zrem(redis_progression_name_set, member_set[0][0])
+            server.zadd(redis_progression_name_set, float(keyword_total_sum), keyword)
+            print redis_progression_name_set
 
 
-def compute_provider_info(server, path):
+def compute_provider_info(server_trend, server_pasteName, path):
+    redis_all_provider = 'all_provider_set'
     
-    redis_avg_size_name_set = 'top_size_set'
-    redis_providers_name_set = 'providers_set'
     paste = Paste.Paste(path)
     
+    paste_baseName = paste.p_name.split('.')[0]
     paste_size = paste._get_p_size()
     paste_provider = paste.p_source
-    paste_date = paste._get_p_date()
-    new_avg = paste_size
+    paste_date = str(paste._get_p_date())
+    redis_sum_size_set = 'top_size_set_' + paste_date
+    redis_avg_size_name_set = 'top_avg_size_set_' + paste_date
+    redis_providers_name_set = 'providers_set_' + paste_date
 
     # Add/Update in Redis
-    prev_num_paste = server.hget(paste_provider+'_num', paste_date)
-    if prev_num_paste is not None:
-        ok = server.hset(paste_provider+'_num', paste_date, int(prev_num_paste)+1)
-        prev_sum_size = server.hget(paste_provider+'_size', paste_date)
-        
-        if prev_sum_size is not None:
-            ok = server.hset(paste_provider+'_size', paste_date, float(prev_sum_size)+paste_size)
-            new_avg = (float(prev_sum_size)+paste_size) / (int(prev_num_paste)+1)
-        else:
-            ok = server.hset(paste_provider+'_size', paste_date, paste_size)
+    server_pasteName.sadd(paste_baseName, path)
+    server_trend.sadd(redis_all_provider, paste_provider)
+
+    num_paste = int(server_trend.hincrby(paste_provider+'_num', paste_date, 1))
+    sum_size = float(server_trend.hincrbyfloat(paste_provider+'_size', paste_date, paste_size))
+    new_avg = float(sum_size) / float(num_paste)
+    server_trend.hset(paste_provider +'_avg', paste_date, new_avg)
 
-    else:
-        ok = server.hset(paste_provider+'_num', paste_date, 1)
-        prev_num_paste = 0
 
     #
     # Compute Most Posted
     #
         
     # Size
-    if paste_provider not in server.smembers(redis_avg_size_name_set): # if it is already in the set
-        if (server.scard(redis_avg_size_name_set) < max_set_cardinality):
-            server.sadd(redis_avg_size_name_set, paste_provider)
+    if server_trend.zcard(redis_sum_size_set) < max_set_cardinality or server_trend.zscore(redis_sum_size_set, paste_provider) != "nil":
+        server_trend.zadd(redis_sum_size_set, float(num_paste), paste_provider)
+        server_trend.zadd(redis_avg_size_name_set, float(new_avg), paste_provider)
+    else: #set full capacity
+        member_set = server_trend.zrangebyscore(redis_sum_size_set, '-inf', '+inf', withscores=True, start=0, num=1)
+        # Member set is a list of (value, score) pairs
+        if float(member_set[0][1]) < new_avg:
+            #remove min from set and add the new one
+            print 'Size - adding ' +paste_provider+ '(' +str(new_avg)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')'
+            server_trend.zrem(redis_sum_size_set, member_set[0][0])
+            server_trend.zadd(redis_sum_size_set, float(sum_size), paste_provider)
+            server_trend.zrem(redis_avg_size_name_set, member_set[0][0])
+            server_trend.zadd(redis_avg_size_name_set, float(new_avg), paste_provider)
 
-        else: #set full capacity
-            #Check value for all members
-            member_set = []
-            for provider in server.smembers(redis_avg_size_name_set):
-                curr_avg = 0.0
-                curr_size = server.hget(provider+'_size', paste_date)
-                curr_num = server.hget(provider+'_num', paste_date)
-                if (curr_size is not None) and (curr_num is not None):
-                    curr_avg = float(curr_size) / float(curr_num)
-                member_set.append((provider, curr_avg))
-            member_set.sort(key=lambda tup: tup[1])
-            if member_set[0][1] < new_avg:
-                #remove min from set and add the new one
-                print 'Size - adding ' +paste_provider+ '(' +str(new_avg)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')'
-                server.srem(redis_avg_size_name_set, member_set[0][0])
-                server.sadd(redis_avg_size_name_set, paste_provider)
 
     # Num
-    if paste_provider not in server.smembers(redis_providers_name_set): # if it is already in the set
-        if (server.scard(redis_providers_name_set) < max_set_cardinality):
-            server.sadd(redis_providers_name_set, paste_provider)
+    # if set not full or provider already present
+    if server_trend.zcard(redis_providers_name_set) < max_set_cardinality or server_trend.zscore(redis_providers_name_set, paste_provider) != "nil":
+        server_trend.zadd(redis_providers_name_set, float(num_paste), paste_provider)
+    else: #set at full capacity
+        member_set = server_trend.zrangebyscore(redis_providers_name_set, '-inf', '+inf', withscores=True, start=0, num=1)
+        # Member set is a list of (value, score) pairs
+        if int(member_set[0][1]) < num_paste:
+            #remove min from set and add the new one
+            print 'Num - adding ' +paste_provider+ '(' +str(num_paste)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')'
+            server_trend.zrem(member_set[0][0])
+            server_trend.zadd(redis_providers_name_set, float(num_paste), paste_provider)
 
-        else: #set full capacity
-            #Check value for all members
-            member_set = []
-            for provider in server.smembers(redis_providers_name_set):
-                curr_num = 0
-                curr_num = server.hget(provider+'_num', paste_date)
-                if curr_num is not None:
-                    member_set.append((provider, int(curr_num)))
-            member_set.sort(key=lambda tup: tup[1])
-            if len(member_set) > 0:
-                if member_set[0][1] < int(prev_num_paste)+1:
-                    #remove min from set and add the new one
-                    print 'Num - adding ' +paste_provider+ '(' +str(int(prev_num_paste)+1)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')'
-                    server.srem(redis_providers_name_set, member_set[0][0])
-                    server.sadd(redis_providers_name_set, paste_provider)
 
 if __name__ == '__main__':
     # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
@@ -167,6 +137,11 @@ if __name__ == '__main__':
         port=p.config.get("Redis_Level_DB_Trending", "port"),
         db=p.config.get("Redis_Level_DB_Trending", "db"))
 
+    r_serv_pasteName = redis.StrictRedis(
+        host=p.config.get("Redis_Paste_Name", "host"),
+        port=p.config.get("Redis_Paste_Name", "port"),
+        db=p.config.get("Redis_Paste_Name", "db"))
+
     # Endless loop getting messages from the input queue
     while True:
         # Get one message from the input queue
@@ -183,4 +158,4 @@ if __name__ == '__main__':
             if len(message.split(';')) > 1:
                 compute_most_posted(r_serv_trend, message)
             else:
-                compute_provider_info(r_serv_trend, message)
+                compute_provider_info(r_serv_trend, r_serv_pasteName, message)

bin/SentimentAnalyser.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python2
+# -*-coding:UTF-8 -*
+"""
+    Sentiment analyser module.
+    It takes its inputs from 'global'.
+
+    The content analysed comes from the pastes with length of the line 
+    above a defined threshold removed (get_p_content_with_removed_lines).
+    This is done because NLTK sentences tokemnizer (sent_tokenize) seems to crash
+    for long lines (function _slices_from_text line#1276).
+
+
+    nltk.sentiment.vader module credit:
+        Hutto, C.J. & Gilbert, E.E. (2014). VADER: A Parsimonious Rule-based Model for Sentiment Analysis of Social Media Text. Eighth International Conference on Weblogs and Social Media (ICWSM-14). Ann Arbor, MI, June 2014.
+
+"""
+
+import time
+import datetime
+import calendar
+import redis
+import json
+from pubsublogger import publisher
+from Helper import Process
+from packages import Paste
+
+from nltk.sentiment.vader import SentimentIntensityAnalyzer
+from nltk import tokenize
+
+# Config Variables
+accepted_Mime_type = ['text/plain']
+size_threshold = 250
+line_max_length_threshold = 1000
+
+def Analyse(message, server):
+    path = message
+    paste = Paste.Paste(path)
+
+    # get content with removed line + number of them
+    num_line_removed, p_content = paste.get_p_content_with_removed_lines(line_max_length_threshold)
+    provider = paste.p_source
+    p_date = str(paste._get_p_date())
+    p_MimeType = paste._get_p_encoding()
+
+    # Perform further analysis
+    if p_MimeType == "text/plain":
+        if isJSON(p_content):
+            p_MimeType = "JSON"
+
+    if p_MimeType in accepted_Mime_type:
+
+        the_date = datetime.date(int(p_date[0:4]), int(p_date[4:6]), int(p_date[6:8]))
+        the_time = datetime.datetime.now()
+        the_time = datetime.time(getattr(the_time, 'hour'), 0, 0)
+        combined_datetime = datetime.datetime.combine(the_date, the_time)
+        timestamp = calendar.timegm(combined_datetime.timetuple())
+    
+        sentences = tokenize.sent_tokenize(p_content.decode('utf-8', 'ignore'))
+    
+        if len(sentences) > 0:
+            avg_score = {'neg': 0.0, 'neu': 0.0, 'pos': 0.0, 'compoundPos': 0.0, 'compoundNeg': 0.0}
+            neg_line = 0
+            pos_line = 0
+            sid = SentimentIntensityAnalyzer()
+            for sentence in sentences:
+                 ss = sid.polarity_scores(sentence)
+                 for k in sorted(ss):
+                     if k == 'compound':
+                         if ss['neg'] > ss['pos']:
+                             avg_score['compoundNeg'] += ss[k]
+                             neg_line += 1
+                         else:
+                             avg_score['compoundPos'] += ss[k]
+                             pos_line += 1
+                     else:
+                         avg_score[k] += ss[k]
+    
+    
+            for k in avg_score:
+                if k == 'compoundPos':
+                    avg_score[k] = avg_score[k] / (pos_line if pos_line > 0 else 1)
+                elif k == 'compoundNeg':
+                    avg_score[k] = avg_score[k] / (neg_line if neg_line > 0 else 1)
+                else:
+                    avg_score[k] = avg_score[k] / len(sentences)
+    
+    
+            # In redis-levelDB: {} = set, () = K-V 
+            # {Provider_set -> provider_i}
+            # {Provider_TimestampInHour_i -> UniqID_i}_j
+            # (UniqID_i -> PasteValue_i)
+    
+            server.sadd('Provider_set', provider)
+    
+            provider_timestamp = provider + '_' + str(timestamp)
+            server.incr('UniqID')
+            UniqID = server.get('UniqID')
+            print provider_timestamp, '->', UniqID, 'dropped', num_line_removed, 'lines'
+            server.sadd(provider_timestamp, UniqID)
+            server.set(UniqID, avg_score)
+    else:
+        print 'Dropped:', p_MimeType
+    
+
+def isJSON(content):
+    try:
+        json.loads(content)
+        return True
+
+    except Exception,e:
+        return False
+
+if __name__ == '__main__':
+    # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
+    # Port of the redis instance used by pubsublogger
+    publisher.port = 6380
+    # Script is the default channel used for the modules.
+    publisher.channel = 'Script'
+
+    # Section name in bin/packages/modules.cfg
+    config_section = 'SentimentAnalysis'
+
+    # Setup the I/O queues
+    p = Process(config_section)
+
+    # Sent to the logging a description of the module
+    publisher.info("<description of the module>")
+
+    # REDIS_LEVEL_DB #
+    server = redis.StrictRedis(
+        host=p.config.get("Redis_Level_DB_Sentiment", "host"),
+        port=p.config.get("Redis_Level_DB_Sentiment", "port"),
+        db=p.config.get("Redis_Level_DB_Sentiment", "db"))
+
+    while True:
+        message = p.get_from_set()
+        if message is None:
+            publisher.debug("{} queue is empty, waiting".format(config_section))
+            time.sleep(1)
+            continue
+
+        Analyse(message, server)
+

bin/WebStats.py

@@ -23,16 +23,11 @@ num_day_to_look = 5       # the detection of the progression start num_day_to_lo
 def analyse(server, field_name, date, url_parsed):
     field = url_parsed[field_name]
     if field is not None:
-        prev_score = server.hget(field, date)
-        if prev_score is not None:
-            server.hset(field, date, int(prev_score) + 1)
-
-        else:
-            server.hset(field, date, 1)
-            if field_name == "domain": #save domain in a set for the monthly plot
-                domain_set_name = "domain_set_" + date[0:6]
-                server.sadd(domain_set_name, field)
-                print "added in " + domain_set_name +": "+ field
+        server.hincrby(field, date, 1)
+        if field_name == "domain": #save domain in a set for the monthly plot
+            domain_set_name = "domain_set_" + date[0:6]
+            server.sadd(domain_set_name, field)
+            print "added in " + domain_set_name +": "+ field
 
 def get_date_range(num_day):
     curr_date = datetime.date.today()
@@ -81,9 +76,10 @@ def compute_progression(server, field_name, num_day, url_parsed):
                 #Check value for all members
                 member_set = []
                 for keyw in server.smembers(redis_progression_name_set):
-                    member_set += (keyw, int(server.hget(redis_progression_name, keyw)))
+                    member_set.append((keyw, int(server.hget(redis_progression_name, keyw))))
+                print member_set
                 member_set.sort(key=lambda tup: tup[1])
-                if member_set[0] < keyword_increase:
+                if member_set[0][1] < keyword_increase:
                     #remove min from set and add the new one
                     server.srem(redis_progression_name_set, member_set[0])
                     server.sadd(redis_progression_name_set, keyword)
@@ -106,11 +102,6 @@ if __name__ == '__main__':
     publisher.info("Makes statistics about valid URL")
 
     # REDIS #
-    r_serv1 = redis.StrictRedis(
-        host=p.config.get("Redis_Level_DB", "host"),
-        port=p.config.get("Redis_Level_DB", "port"),
-        db=p.config.get("Redis_Level_DB", "db"))
- 
     r_serv_trend = redis.StrictRedis(
         host=p.config.get("Redis_Level_DB_Trending", "host"),
         port=p.config.get("Redis_Level_DB_Trending", "port"),

bin/empty_queue.py

@@ -24,6 +24,7 @@ if __name__ == "__main__":
     publisher.channel = "Script"
 
     config_section = ['Global', 'Duplicates', 'Indexer', 'Attributes', 'Lines', 'DomClassifier', 'Tokenize', 'Curve', 'Categ', 'CreditCards', 'Mail', 'Onion', 'DumpValidOnion', 'Web',  'WebStats', 'Release', 'Credential', 'Cve', 'Phone', 'SourceCode', 'Keys']
+    config_section = ['Curve']
 
     for queue in config_section:
         print 'dropping: ' + queue

bin/launch_lvldb.sh

@@ -7,13 +7,17 @@ lvdbhost='127.0.0.1'
 lvdbdir="${AIL_HOME}/LEVEL_DB_DATA/"
 db1_y='2013'
 db2_y='2014'
-db2_y='2015'
-db2_y='2016'
+db3_y='2016'
+db4_y='3016'
 nb_db=13
 
 screen -dmS "LevelDB"
 sleep 0.1
 echo -e $GREEN"\t* Launching Levels DB servers"$DEFAULT
-#Add lines here with appropriates options.
-screen -S "LevelDB" -X screen -t "2016" bash -c '../redis-leveldb/redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db2_y' -M '$nb_db'; read x'
+sleep 0.1
+screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db3_y' -M '$nb_db'; read x'
+
+# For Curve
+sleep 0.1
+screen -S "LevelDB" -X screen -t "3016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'3016/ -P '$db4_y' -M '$nb_db'; read x'
 

bin/launch_redis.sh

@@ -14,3 +14,6 @@ screen -S "Redis" -X screen -t "6380" bash -c '../redis/src/redis-server '$conf_
 sleep 0.1
 screen -S "Redis" -X screen -t "6381" bash -c '../redis/src/redis-server '$conf_dir'6381.conf ; read x'
 
+# For Words and curves
+sleep 0.1
+screen -S "Redis" -X screen -t "6382" bash -c '../redis/src/redis-server '$conf_dir'6382.conf ; read x'

bin/launch_scripts.sh

@@ -9,8 +9,8 @@ sleep 0.1
 echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
 
 screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x'
-#sleep 0.1
-#screen -S "Script" -X screen -t "Duplicate" bash -c './Duplicate.py; read x'
+sleep 0.1
+screen -S "Script" -X screen -t "Duplicate" bash -c './Duplicate_ssdeep_v2.py; read x'
 sleep 0.1
 screen -S "Script" -X screen -t "Attribute" bash -c './Attribute.py; read x'
 sleep 0.1
@@ -34,6 +34,24 @@ screen -S "Script" -X screen -t "Credential" bash -c './Credential.py; read x'
 sleep 0.1
 screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x'
 sleep 0.1
+screen -S "Script" -X screen -t "Curve_topsets_manager" bash -c './Curve_manage_top_sets.py; read x'
+sleep 0.1
 screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x'
 sleep 0.1
+screen -S "Script" -X screen -t "Keys" bash -c './Keys.py; read x'
+sleep 0.1
+screen -S "Script" -X screen -t "Phone" bash -c './Phone.py; read x'
+sleep 0.1
+screen -S "Script" -X screen -t "Release" bash -c './Release.py; read x'
+sleep 0.1
+screen -S "Script" -X screen -t "Cve" bash -c './Cve.py; read x'
+sleep 0.1
 screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x'
+sleep 0.1
+screen -S "Script" -X screen -t "ModuleStats" bash -c './ModuleStats.py; read x'
+sleep 0.1
+screen -S "Script" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x'
+sleep 0.1
+screen -S "Script" -X screen -t "Browse_warning_paste" bash -c './Browse_warning_paste.py; read x'
+sleep 0.1
+screen -S "Script" -X screen -t "SentimentAnalyser" bash -c './SentimentAnalyser.py; read x'

bin/packages/Paste.py

@@ -91,6 +91,7 @@ class Paste(object):
         self.p_langage = None
         self.p_nb_lines = None
         self.p_max_length_line = None
+        self.array_line_above_threshold = None
         self.p_duplicate = None
 
     def get_p_content(self):
@@ -118,6 +119,21 @@ class Paste(object):
     def get_p_content_as_file(self):
         return cStringIO.StringIO(self.get_p_content())
 
+    def get_p_content_with_removed_lines(self, threshold):
+        num_line_removed = 0
+        line_length_threshold = threshold
+        string_content = ""
+        f = self.get_p_content_as_file()
+        line_id = 0
+        for line_id, line in enumerate(f):
+            length = len(line)
+            if length < line_length_threshold:
+                string_content += line
+            else:
+                num_line_removed+=1
+
+        return (num_line_removed, string_content)
+
     def get_lines_info(self):
         """
         Returning and setting the number of lines and the maximum lenght of the
@@ -136,9 +152,11 @@ class Paste(object):
                 length = len(line)
                 if length >= max_length_line:
                     max_length_line = length
+
             f.close()
             self.p_nb_lines = line_id
             self.p_max_length_line = max_length_line
+
         return (self.p_nb_lines, self.p_max_length_line)
 
     def _get_p_encoding(self):

bin/packages/config.cfg.sample

@@ -1,6 +1,5 @@
 [Directories]
 bloomfilters = Blooms
-#Duplicate_ssdeep
 dicofilters = Dicos
 pastes = PASTES
 
@@ -59,11 +58,26 @@ host = localhost
 port = 6379
 db = 1
 
+[Redis_Paste_Name]
+host = localhost
+port = 6379
+db = 2
+
 ##### LevelDB #####
 [Redis_Level_DB_Curve]
 host = localhost
-port = 3016
-db = 0
+port = 6382
+db = 1
+
+[Redis_Level_DB_Sentiment]
+host = localhost
+port = 6382
+db = 4
+
+[Redis_Level_DB_TermFreq]
+host = localhost
+port = 6382
+db = 2
 
 [Redis_Level_DB]
 host = localhost
@@ -72,8 +86,8 @@ db = 0
 
 [Redis_Level_DB_Trending]
 host = localhost
-port = 2016
-db = 0
+port = 6382
+db = 3
 
 [Redis_Level_DB_Hashs]
 host = localhost

bin/packages/modules.cfg

@@ -24,6 +24,10 @@ publish = Redis_Words
 
 [Curve]
 subscribe = Redis_Words
+publish = Redis_CurveManageTopSets
+
+[CurveManageTopSets]
+subscribe = Redis_CurveManageTopSets
 
 [Categ]
 subscribe = Redis_Global
@@ -66,6 +70,9 @@ subscribe = Redis_BrowseWarningPaste
 #subscribe = Redis_Cve
 #publish = Redis_BrowseWarningPaste
 
+[SentimentAnalysis]
+subscribe = Redis_Global
+
 [Release]
 subscribe = Redis_Global
 

installing_deps.sh

@@ -42,7 +42,7 @@ popd
 # tlsh
 test ! -d tlsh && git clone git://github.com/trendmicro/tlsh.git
 pushd tlsh/
-./make
+./make.sh
 popd
 
 # REDIS LEVEL DB #

var/www/Flask_server.py

@@ -5,6 +5,8 @@ import redis
 import ConfigParser
 import json
 import datetime
+import time
+import calendar
 from flask import Flask, render_template, jsonify, request
 import flask
 import os
@@ -49,6 +51,22 @@ r_serv_db = redis.StrictRedis(
     port=cfg.getint("Redis_Level_DB", "port"),
     db=cfg.getint("Redis_Level_DB", "db"))
 
+r_serv_sentiment = redis.StrictRedis(
+        host=cfg.get("Redis_Level_DB_Sentiment", "host"),
+        port=cfg.getint("Redis_Level_DB_Sentiment", "port"),
+        db=cfg.getint("Redis_Level_DB_Sentiment", "db"))
+
+r_serv_term = redis.StrictRedis(
+        host=cfg.get("Redis_Level_DB_TermFreq", "host"),
+        port=cfg.getint("Redis_Level_DB_TermFreq", "port"),
+        db=cfg.getint("Redis_Level_DB_TermFreq", "db"))
+
+r_serv_pasteName = redis.StrictRedis(
+    host=cfg.get("Redis_Paste_Name", "host"),
+    port=cfg.getint("Redis_Paste_Name", "port"),
+    db=cfg.getint("Redis_Paste_Name", "db"))
+
+
 app = Flask(__name__, static_url_path='/static/')
 
 
@@ -106,7 +124,7 @@ def parseStringToList2(the_string):
         return res
 
 
-def showpaste(content_range):    
+def showpaste(content_range):
     requested_path = request.args.get('paste', '')
     paste = Paste.Paste(requested_path)
     p_date = str(paste._get_p_date())
@@ -129,7 +147,7 @@ def showpaste(content_range):
             dup_list[2] = int(((tlsh_to_percent - float(dup_list[2])) / tlsh_to_percent)*100)
         else:
             dup_list[2] = int(dup_list[2])
-            
+
     p_duplicate_full_list.sort(lambda x,y: cmp(x[2], y[2]), reverse=True)
 
     # Combine multiple duplicate paste name and format for display
@@ -158,7 +176,8 @@ def showpaste(content_range):
         p_hashtype_list.append(hash_type)
 
     if content_range != 0:
-       p_content = p_content[0:content_range] 
+       p_content = p_content[0:content_range]
+
 
     return render_template("show_saved_paste.html", date=p_date, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=len(p_content), duplicate_list = p_duplicate_list, simil_list = p_simil_list, hashtype_list = p_hashtype_list)
 
@@ -181,23 +200,32 @@ def get_date_range(num_day):
 # Iterate over elements in the module provided and return the today data or the last data
 # return format: [('passed_days', num_of_passed_days), ('elem_name1', elem_value1), ('elem_name2', elem_value2)]]
 def get_top_relevant_data(server, module_name):
-    redis_progression_name_set = 'top_'+ module_name +'_set'
-    days = 0 
+    days = 0
     for date in get_date_range(15):
-        member_set = []
-        for keyw in server.smembers(redis_progression_name_set):
-            redis_progression_name = module_name+'-'+keyw
-            keyw_value = server.hget(date ,redis_progression_name)
-            keyw_value = keyw_value if keyw_value is not None else 0
-            member_set.append((keyw, int(keyw_value)))
-        member_set.sort(key=lambda tup: tup[1], reverse=True)
-        if member_set[0][1] == 0: #No data for this date
+        redis_progression_name_set = 'top_'+ module_name +'_set_' + date
+        member_set = server.zrevrangebyscore(redis_progression_name_set, '+inf', '-inf', withscores=True)
+        if len(member_set) == 0: #No data for this date
             days += 1
-            continue
         else:
             member_set.insert(0, ("passed_days", days))
             return member_set
 
+
+def Term_getValueOverRange(word, startDate, num_day):
+    passed_days = 0
+    oneDay = 60*60*24
+    to_return = []
+    curr_to_return = 0
+    for timestamp in range(startDate, startDate - max(num_day)*oneDay, -oneDay):
+        value = r_serv_term.hget(timestamp, word)
+        curr_to_return += int(value) if value is not None else 0
+        for i in num_day:
+            if passed_days == i-1:
+                to_return.append(curr_to_return)
+        passed_days += 1
+    return to_return
+
+
 # ========= CACHE CONTROL ========
 @app.after_request
 def add_header(response):
@@ -225,19 +253,19 @@ def progressionCharts():
     attribute_name = request.args.get('attributeName')
     trending_name = request.args.get('trendingName')
     bar_requested = True if request.args.get('bar') == "true" else False
-    
+
     if (bar_requested):
         num_day = int(request.args.get('days'))
         bar_values = []
 
-	date_range = get_date_range(num_day) 
+        date_range = get_date_range(num_day)
         # Retreive all data from the last num_day
         for date in date_range:
             curr_value = r_serv_charts.hget(attribute_name, date)
             bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], int(curr_value if curr_value is not None else 0)])
         bar_values.insert(0, attribute_name)
         return jsonify(bar_values)
- 
+
     else:
         redis_progression_name = 'top_progression_'+trending_name
         redis_progression_name_set = 'top_progression_'+trending_name+'_set'
@@ -252,25 +280,25 @@ def progressionCharts():
         if len(member_set) == 0:
             member_set.append(("No relevant data", int(100)))
         return jsonify(member_set)
-    
+
 @app.route("/_moduleCharts", methods=['GET'])
 def modulesCharts():
     keyword_name = request.args.get('keywordName')
     module_name = request.args.get('moduleName')
     bar_requested = True if request.args.get('bar') == "true" else False
-    
+
     if (bar_requested):
         num_day = int(request.args.get('days'))
         bar_values = []
 
-	date_range = get_date_range(num_day) 
+        date_range = get_date_range(num_day)
         # Retreive all data from the last num_day
         for date in date_range:
             curr_value = r_serv_charts.hget(date, module_name+'-'+keyword_name)
             bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], int(curr_value if curr_value is not None else 0)])
         bar_values.insert(0, keyword_name)
         return jsonify(bar_values)
- 
+
     else:
         member_set = get_top_relevant_data(r_serv_charts, module_name)
         if len(member_set) == 0:
@@ -283,52 +311,33 @@ def providersChart():
     keyword_name = request.args.get('keywordName')
     module_name = request.args.get('moduleName')
     bar_requested = True if request.args.get('bar') == "true" else False
-    
+
     if (bar_requested):
         num_day = int(request.args.get('days'))
         bar_values = []
 
-	date_range = get_date_range(num_day) 
+        date_range = get_date_range(num_day)
         # Retreive all data from the last num_day
         for date in date_range:
             curr_value_size = r_serv_charts.hget(keyword_name+'_'+'size', date)
             curr_value_num = r_serv_charts.hget(keyword_name+'_'+'num', date)
+            curr_value_size_avg = r_serv_charts.hget(keyword_name+'_'+'avg', date)
             if module_name == "size":
-                curr_value_num = curr_value_num if curr_value_num is not None else 0
-                curr_value_num = curr_value_num if int(curr_value_num) != 0 else 10000000000
-                curr_value = float(curr_value_size if curr_value_size is not None else 0.0) / float(curr_value_num)
+                curr_value = float(curr_value_size_avg if curr_value_size_avg is not None else 0)
             else:
                 curr_value = float(curr_value_num if curr_value_num is not None else 0.0)
 
             bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], curr_value])
         bar_values.insert(0, keyword_name)
         return jsonify(bar_values)
- 
+
     else:
-        redis_provider_name_set = 'top_size_set' if module_name == "size" else 'providers_set'
-
-        # Iterate over element in top_x_set and retreive their value
-        member_set = []
-        for keyw in r_serv_charts.smembers(redis_provider_name_set):
-            redis_provider_name_size = keyw+'_'+'size'
-            redis_provider_name_num = keyw+'_'+'num'
-            keyw_value_size = r_serv_charts.hget(redis_provider_name_size, get_date_range(0)[0])
-            keyw_value_size = keyw_value_size if keyw_value_size is not None else 0.0
-            keyw_value_num = r_serv_charts.hget(redis_provider_name_num, get_date_range(0)[0])
-            
-            if keyw_value_num is not None:
-                keyw_value_num = int(keyw_value_num)
-            else:
-                if module_name == "size":
-                    keyw_value_num = 10000000000
-                else:
-                    keyw_value_num = 0
-            if module_name == "size":
-                member_set.append((keyw, float(keyw_value_size)/float(keyw_value_num)))
-            else:
-                member_set.append((keyw, float(keyw_value_num)))
-
-        member_set.sort(key=lambda tup: tup[1], reverse=True)
+        #redis_provider_name_set = 'top_size_set' if module_name == "size" else 'providers_set'
+        redis_provider_name_set = 'top_avg_size_set_' if module_name == "size" else 'providers_set_'
+        redis_provider_name_set = redis_provider_name_set + get_date_range(0)[0]
+        
+        member_set = r_serv_charts.zrevrangebyscore(redis_provider_name_set, '+inf', '-inf', withscores=True, start=0, num=8)
+        # Member set is a list of (value, score) pairs
         if len(member_set) == 0:
             member_set.append(("No relevant data", float(100)))
         return jsonify(member_set)
@@ -344,7 +353,22 @@ def search():
     c = [] #preview of the paste content
     paste_date = []
     paste_size = []
-    # Search
+
+    # Search filename
+    print r_serv_pasteName.smembers(q[0])
+    for path in r_serv_pasteName.smembers(q[0]):
+        print path
+        r.append(path)
+        paste = Paste.Paste(path)
+        content = paste.get_p_content().decode('utf8', 'ignore')
+        content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
+        c.append(content[0:content_range])
+        curr_date = str(paste._get_p_date())
+        curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
+        paste_date.append(curr_date)
+        paste_size.append(paste._get_p_size())
+
+    # Search full line
     from whoosh import index
     from whoosh.fields import Schema, TEXT, ID
     schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT)
@@ -360,11 +384,11 @@ def search():
             paste = Paste.Paste(x.items()[0][1])
             content = paste.get_p_content().decode('utf8', 'ignore')
             content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
-            c.append(content[0:content_range]) 
+            c.append(content[0:content_range])
             curr_date = str(paste._get_p_date())
             curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
-            paste_date.append(curr_date) 
-            paste_size.append(paste._get_p_size()) 
+            paste_date.append(curr_date)
+            paste_size.append(paste._get_p_size())
     return render_template("search.html", r=r, c=c, query=request.form['query'], paste_date=paste_date, paste_size=paste_size, char_to_display=max_preview_modal)
 
 
@@ -417,11 +441,11 @@ def importantPasteByModule():
         paste = Paste.Paste(path)
         content = paste.get_p_content().decode('utf8', 'ignore')
         content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
-        all_content.append(content[0:content_range]) 
+        all_content.append(content[0:content_range])
         curr_date = str(paste._get_p_date())
         curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
-        paste_date.append(curr_date) 
-        paste_linenum.append(paste.get_lines_info()[0]) 
+        paste_date.append(curr_date)
+        paste_linenum.append(paste.get_lines_info()[0])
 
     return render_template("important_paste_by_module.html", all_path=all_path, content=all_content, paste_date=paste_date, paste_linenum=paste_linenum, char_to_display=max_preview_modal)
 
@@ -429,6 +453,299 @@ def importantPasteByModule():
 def moduletrending():
     return render_template("Moduletrending.html")
 
+@app.route("/sentiment_analysis_trending/")
+def sentiment_analysis_trending():
+    return render_template("sentiment_analysis_trending.html")
+
+
+@app.route("/sentiment_analysis_getplotdata/")
+def sentiment_analysis_getplotdata():
+    # Get the top providers based on number of pastes
+    oneHour = 60*60
+    sevenDays = oneHour*24*7
+    dateStart = datetime.datetime.now()
+    dateStart = dateStart.replace(minute=0, second=0, microsecond=0)
+    dateStart_timestamp = calendar.timegm(dateStart.timetuple())
+
+    to_return = {}
+    range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(0)[0], '+inf', '-inf', start=0, num=8)
+    # if empty, get yesterday top providers
+    print 'providers_set_'+ get_date_range(1)[1]
+    range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(1)[1], '+inf', '-inf', start=0, num=8) if range_providers == [] else range_providers
+    # if still empty, takes from all providers
+    if range_providers == []:
+        print 'today provider empty'
+        range_providers = r_serv_charts.smembers('all_provider_set')
+
+    for cur_provider in range_providers:
+        print cur_provider
+        cur_provider_name = cur_provider + '_'
+        list_date = {}
+        for cur_timestamp in range(int(dateStart_timestamp), int(dateStart_timestamp)-sevenDays-oneHour, -oneHour):
+            cur_set_name = cur_provider_name + str(cur_timestamp)
+
+            list_value = []
+            for cur_id in r_serv_sentiment.smembers(cur_set_name):
+                cur_value = r_serv_sentiment.get(cur_id)
+                list_value.append(cur_value)
+            list_date[cur_timestamp] = list_value
+        to_return[cur_provider] = list_date
+
+    return jsonify(to_return)
+
+
+
+@app.route("/sentiment_analysis_plot_tool/")
+def sentiment_analysis_plot_tool():
+    return render_template("sentiment_analysis_plot_tool.html")
+
+
+
+@app.route("/sentiment_analysis_plot_tool_getdata/", methods=['GET'])
+def sentiment_analysis_plot_tool_getdata():
+    getProviders = request.args.get('getProviders')
+
+    if getProviders == 'True':
+        providers = []
+        for cur_provider in r_serv_charts.smembers('all_provider_set'):
+            providers.append(cur_provider)
+        return jsonify(providers)
+
+    else:
+        query = request.args.get('query')
+        query = query.split(',')
+        Qdate = request.args.get('Qdate')
+
+        date1 = (Qdate.split('-')[0]).split('.')
+        date1 = datetime.date(int(date1[2]), int(date1[1]), int(date1[0]))
+
+        date2 = (Qdate.split('-')[1]).split('.')
+        date2 = datetime.date(int(date2[2]), int(date2[1]), int(date2[0]))
+
+        timestamp1 = calendar.timegm(date1.timetuple())
+        timestamp2 = calendar.timegm(date2.timetuple())
+
+        oneHour = 60*60
+        oneDay = oneHour*24
+
+        to_return = {}
+        for cur_provider in query:
+            list_date = {}
+            cur_provider_name = cur_provider + '_'
+            for cur_timestamp in range(int(timestamp1), int(timestamp2)+oneDay, oneHour):
+                cur_set_name = cur_provider_name + str(cur_timestamp)
+
+                list_value = []
+                for cur_id in r_serv_sentiment.smembers(cur_set_name):
+                    cur_value = r_serv_sentiment.get(cur_id)
+                    list_value.append(cur_value)
+                list_date[cur_timestamp] = list_value
+            to_return[cur_provider] = list_date
+
+        return jsonify(to_return)
+
+
+@app.route("/terms_management/")
+def terms_management():
+    TrackedTermsSet_Name = "TrackedSetTermSet"
+    BlackListTermsSet_Name = "BlackListSetTermSet"
+    TrackedTermsDate_Name = "TrackedTermDate"
+    BlackListTermsDate_Name = "BlackListTermDate"
+    
+    today = datetime.datetime.now()
+    today = today.replace(hour=0, minute=0, second=0, microsecond=0)
+    today_timestamp = calendar.timegm(today.timetuple())
+
+    track_list = []
+    track_list_values = []
+    track_list_num_of_paste = []
+    for tracked_term in r_serv_term.smembers(TrackedTermsSet_Name):
+        track_list.append(tracked_term)
+        value_range = Term_getValueOverRange(tracked_term, today_timestamp, [1, 7, 31])
+
+        term_date = r_serv_term.hget(TrackedTermsDate_Name, tracked_term)
+
+        set_paste_name = "tracked_" + tracked_term
+        track_list_num_of_paste.append(r_serv_term.scard(set_paste_name))
+        term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded"
+        value_range.append(term_date)
+        track_list_values.append(value_range)
+
+
+    black_list = []
+    for blacked_term in r_serv_term.smembers(BlackListTermsSet_Name):
+        term_date = r_serv_term.hget(BlackListTermsDate_Name, blacked_term)
+        term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded"
+        black_list.append([blacked_term, term_date])
+
+    return render_template("terms_management.html", black_list=black_list, track_list=track_list, track_list_values=track_list_values,  track_list_num_of_paste=track_list_num_of_paste)
+
+
+@app.route("/terms_management_query_paste/")
+def terms_management_query_paste():
+    term =  request.args.get('term')
+    TrackedTermsSet_Name = "TrackedSetTermSet"
+    paste_info = []
+
+    set_paste_name = "tracked_" + term
+    track_list_path = r_serv_term.smembers(set_paste_name)
+
+    for path in track_list_path:
+        paste = Paste.Paste(path)
+        p_date = str(paste._get_p_date())
+        p_date = p_date[6:]+'/'+p_date[4:6]+'/'+p_date[0:4]
+        p_source = paste.p_source
+        p_encoding = paste._get_p_encoding()
+        p_size = paste.p_size
+        p_mime = paste.p_mime
+        p_lineinfo = paste.get_lines_info()
+        p_content = paste.get_p_content().decode('utf-8', 'ignore')
+        if p_content != 0:
+            p_content = p_content[0:400]
+        paste_info.append({"path": path, "date": p_date, "source": p_source, "encoding": p_encoding, "size": p_size, "mime": p_mime, "lineinfo": p_lineinfo, "content": p_content})
+
+    return jsonify(paste_info)
+
+
+@app.route("/terms_management_query/")
+def terms_management_query():
+    TrackedTermsDate_Name = "TrackedTermDate"
+    BlackListTermsDate_Name = "BlackListTermDate"
+    term =  request.args.get('term')
+    section = request.args.get('section')
+
+    today = datetime.datetime.now()
+    today = today.replace(hour=0, minute=0, second=0, microsecond=0)
+    today_timestamp = calendar.timegm(today.timetuple())
+    value_range = Term_getValueOverRange(term, today_timestamp, [1, 7, 31])
+
+    if section == "followTerm":
+        term_date = r_serv_term.hget(TrackedTermsDate_Name, term)
+    elif section == "blacklistTerm":
+        term_date = r_serv_term.hget(BlackListTermsDate_Name, term)
+
+    term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded"
+    value_range.append(str(term_date))
+    return jsonify(value_range)
+
+
+@app.route("/terms_management_action/", methods=['GET'])
+def terms_management_action():
+    TrackedTermsSet_Name = "TrackedSetTermSet"
+    TrackedTermsDate_Name = "TrackedTermDate"
+    BlackListTermsDate_Name = "BlackListTermDate"
+    BlackListTermsSet_Name = "BlackListSetTermSet"
+
+    today = datetime.datetime.now()
+    today = today.replace(microsecond=0)
+    today_timestamp = calendar.timegm(today.timetuple())
+
+
+    section = request.args.get('section')
+    action = request.args.get('action')
+    term =  request.args.get('term')
+    if action is None or term is None:
+        return "None"
+    else:
+        if section == "followTerm":
+            if action == "add":
+                r_serv_term.sadd(TrackedTermsSet_Name, term.lower())
+                r_serv_term.hset(TrackedTermsDate_Name, term, today_timestamp)
+            else:
+                r_serv_term.srem(TrackedTermsSet_Name, term.lower())
+        elif section == "blacklistTerm":
+            if action == "add":
+                r_serv_term.sadd(BlackListTermsSet_Name, term.lower())
+                r_serv_term.hset(BlackListTermsDate_Name, term, today_timestamp)
+            else:
+                r_serv_term.srem(BlackListTermsSet_Name, term.lower())
+        else:
+            return "None"
+
+        to_return = {}
+        to_return["section"] = section
+        to_return["action"] = action
+        to_return["term"] = term
+        return jsonify(to_return)
+
+
+
+@app.route("/terms_plot_tool/")
+def terms_plot_tool():
+    term =  request.args.get('term')
+    if term is not None:
+        return render_template("terms_plot_tool.html", term=term)
+    else:
+        return render_template("terms_plot_tool.html", term="")
+
+
+@app.route("/terms_plot_tool_data/")
+def terms_plot_tool_data():
+    oneDay = 60*60*24
+    range_start =  datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_start')))) if request.args.get('range_start') is not None else 0;
+    range_start = range_start.replace(hour=0, minute=0, second=0, microsecond=0)
+    range_start = calendar.timegm(range_start.timetuple())
+    range_end =  datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_end')))) if request.args.get('range_end') is not None else 0;
+    range_end = range_end.replace(hour=0, minute=0, second=0, microsecond=0)
+    range_end = calendar.timegm(range_end.timetuple())
+    term =  request.args.get('term')
+
+    if term is None:
+        return "None"
+    else:
+        value_range = []
+        for timestamp in range(range_start, range_end+oneDay, oneDay):
+            print timestamp, term
+            value = r_serv_term.hget(timestamp, term)
+            curr_value_range = int(value) if value is not None else 0
+            value_range.append([timestamp, curr_value_range])
+        return jsonify(value_range)
+
+
+@app.route("/terms_plot_top/")
+def terms_plot_top():
+    return render_template("terms_plot_top.html")
+
+
+@app.route("/terms_plot_top_data/")
+def terms_plot_top_data():
+    oneDay = 60*60*24
+    today = datetime.datetime.now()
+    today = today.replace(hour=0, minute=0, second=0, microsecond=0)
+    today_timestamp = calendar.timegm(today.timetuple())
+
+    set_day = "TopTermFreq_set_day_" + str(today_timestamp)
+    set_week = "TopTermFreq_set_week";
+    set_month = "TopTermFreq_set_month";
+
+    the_set = request.args.get('set')
+    num_day = int(request.args.get('num_day'))
+    if the_set is None:
+        return "None"
+    else:
+        to_return = []
+        if the_set == "TopTermFreq_set_day":
+            the_set += "_" + str(today_timestamp)
+
+        for term, tot_value in r_serv_term.zrevrangebyscore(the_set, '+inf', '-inf', withscores=True, start=0, num=20):
+            position = {}
+            position['day'] = r_serv_term.zrevrank(set_day, term)
+            position['day'] = position['day']+1 if position['day'] is not None else "<20"
+            position['week'] = r_serv_term.zrevrank(set_week, term)
+            position['week'] = position['week']+1 if position['week'] is not None else "<20"
+            position['month'] = r_serv_term.zrevrank(set_month, term)
+            position['month'] = position['month']+1 if position['month'] is not None else "<20"
+            value_range = []
+            for timestamp in range(today_timestamp, today_timestamp - num_day*oneDay, -oneDay):
+                value = r_serv_term.hget(timestamp, term)
+                curr_value_range = int(value) if value is not None else 0
+                value_range.append([timestamp, curr_value_range])
+                
+            to_return.append([term, value_range, tot_value, position])
+    
+        return jsonify(to_return)
+
+
 
 @app.route("/showsavedpaste/") #completely shows the paste in a new tab
 def showsavedpaste():
@@ -446,7 +763,7 @@ def getmoredata():
     paste = Paste.Paste(requested_path)
     p_content = paste.get_p_content().decode('utf-8', 'ignore')
     to_return = p_content[max_preview_modal-1:]
-    return to_return 
+    return to_return
 
 
 if __name__ == "__main__":

var/www/static/css/switch_checkbox.css

@@ -0,0 +1,68 @@
+.switch {
+    position: relative;
+    display: inline-block;
+    vertical-align: top;
+    width: 56px;
+    height: 20px;
+    padding: 3px;
+    margin-left: 5px;
+    border-radius: 18px;
+    box-shadow: inset 0 -1px #fff,inset 0 1px 1px rgba(0,0,0,0.05);
+    cursor: pointer;
+    -webkit-box-sizing: content-box;
+    -moz-box-sizing: content-box;
+    box-sizing: content-box;
+}
+
+.switch-input {
+    position: absolute;
+    top: 0;
+    left: 0;
+    opacity: 0;
+}
+
+.switch-input:checked ~ .switch-label {
+    background: #67c2ef;
+    box-shadow: inset 0 1px 2px rgba(0,0,0,0.15),inset 0 0 3px rgba(0,0,0,0.2);
+}
+.switch-label {
+    position: relative;
+    display: block;
+    height: inherit;
+    font-size: 10px;
+    text-transform: uppercase;
+    background: #f9f9f9;
+    border-radius: inherit;
+    box-shadow: inset 0 1px 2px rgba(0,0,0,0.12),inset 0 0 2px rgba(0,0,0,0.15);
+    -webkit-transition: .15s ease-out;
+    -moz-transition: .15s ease-out;
+    -o-transition: .15s ease-out;
+    transition: .15s ease-out;
+    -webkit-transition-property: opacity background;
+    -moz-transition-property: opacity background;
+    -o-transition-property: opacity background;
+    transition-property: opacity background;
+}
+
+.switch-input:checked ~ .switch-handle {
+    left: 40px;
+    box-shadow: -1px 1px 5px rgba(0,0,0,0.2);
+}
+.switch-handle {
+    position: absolute;
+    top: 4px;
+    left: 4px;
+    width: 18px;
+    height: 18px;
+    background: white;
+    border-radius: 10px;
+    box-shadow: 1px 1px 5px rgba(0,0,0,0.2);
+    background-image: -webkit-linear-gradient(top,#fff 40%,#f0f0f0);
+    background-image: -moz-linear-gradient(top,#fff 40%,#f0f0f0);
+    background-image: -o-linear-gradient(top,#fff 40%,#f0f0f0);
+    background-image: linear-gradient(to bottom,#fff 40%,#f0f0f0);
+    -webkit-transition: left .15s ease-out;
+    -moz-transition: left .15s ease-out;
+    -o-transition: left .15s ease-out;
+    transition: left .15s ease-out;
+}

var/www/static/js/FlexGauge.js

@@ -0,0 +1,440 @@
+/**
+ *  FlexGauge
+ *  Version: 1.0
+ *  Author: Jeff Millies
+ *  Author URI:
+ *
+ *  Slight modification for better display in Sentiment webpages
+ */
+(function ($) {
+    var FlexGauge = function (o) {
+        if (typeof o === 'object') {
+            this._extendOptions(o, false);
+            this._build();
+        }
+    };
+    FlexGauge.prototype = {
+        /**
+         *  {String} Element that you would like to append to. ie '#idname', '.classname', 'div#idname', etc..
+         */
+        appendTo: 'body',
+        /**
+         *  {String} Id of Canvas already created or Id of canvas that will be created automatically
+         */
+        elementId: 'canvas',
+        /**
+         *  {String} Class of canvas created
+         */
+        elementClass: 'canvas',
+        /**
+         *  {Int} Canvas Width & Height
+         */
+        elementWidth: 200,
+        elementHeight: 200,
+        /**
+         *  {Boolean|String} Generate Dial Value for the Gauge, true will use arcFillPercent or arcFillInt
+         *  depending on provided values and specified dialUnits, string will use specified value
+         */
+        dialValue: false,
+        /**
+         *  {String} Class applied to div when dial is generated.
+         */
+        dialClass: 'fg-dial',
+        /**
+         *  {string: %|$| } Type of unit to use for the dial
+         */
+        dialUnit: '%',
+        /**
+         *  {string: before|after} Where the dial unit will be displayed
+         */
+        dialUnitPosition: 'after',
+        /**
+         *  {Boolean|String} Generate Label for the Gauge, true will use default "FlexGauge", string will use specified
+         */
+        dialLabel: false,
+        /**
+         *  {String} Class applied to div when label is generated.
+         */
+        dialLabelClass: 'fg-dial-label',
+        /**
+         *  {Int} Radius of the arc
+         */
+        inc: 0.0,
+        incTot: 1.0,
+        /**
+         *  {Doule} Increment value
+         */
+        arcSize: 85,
+        /**
+         *  {double} Starting and Ending location of the arc, End always needs to be larger
+         *  arc(x, y, radius, startAngle, endAngle, anticlockwise)
+         */
+        arcAngleStart: 0.85,
+        arcAngleEnd: 2.15,
+        /**
+         *  {double} Percentage the arc fills
+         */
+        arcFillPercent: .5,
+        /**
+         *  {Int} Starting and Ending values that are used to
+         *  find a difference for amount of units
+         *  ie: 60 (arcFillEnd) - 10 (arcFillStart) = 50
+         */
+        arcFillStart: null,
+        arcFillEnd: null,
+        /**
+         *  {Int} Data used to find out what percentage of the
+         *  arc to fill. arcFillInt can be populated by
+         *  the difference of arcFillStart and arcFillEnd
+         */
+        arcFillInt: null,
+        arcFillTotal: null,
+        /**
+         *  {Int} Color lightness: 0 - 255, 0 having no white added, 255 having all white and no color
+         */
+        arcBgColorLight: 80,
+        /**
+         *  {Int} Color saturation: 0 - 100, 0 having no color, 100 is full color
+         */
+        arcBgColorSat: 60,
+        /**
+         *  {Int} Size of the line marking the percentage
+         */
+        arcStrokeFg: 30,
+        /**
+         *  {Int} Size of the container holding the line
+         */
+        arcStrokeBg: 30,
+
+        /**
+         *  {string: hex} Color of the line marking the percentage
+         */
+        colorArcFg: '#5bc0de',
+        /**
+         *  {string: hex} Color of the container holding the line, default is using the Fg color and lightening it
+         */
+        colorArcBg: null,
+
+        /**
+         *  {String} Instead of providing a color or hex for the color, you can provide a class from the style
+         *  sheet and specify what you would like to grab for the color in styleSrc
+         */
+        styleArcFg: null,
+        styleArcBg: null,
+        styleSrc: 'color',
+
+        /**
+         *  {Boolean} If set to false, then the graph will not be animated
+         */
+        animateEasing: true,
+        /**
+         *  {Int} Speed for the animation, 1 is fastest, higher the number, slower the animation
+         */
+        animateSpeed: 5,
+        /**
+         *  {Int} Math used in animation speed
+         */
+        animateNumerator: 12,
+        animateDivisor: 15,
+
+        /**
+         *  {double} Placeholder for current percentage while animating
+         */
+        _animatePerc: 0.00,
+
+        /**
+         *  {Object} Placeholder for setInterval
+         */
+        _animateLoop: null,
+
+        /**
+         *  {Object} Placeholder for canvas
+         */
+        _canvas: null,
+
+        /**
+         *  {Object} Placeholder for canvas context
+         */
+        _ctx: null,
+
+        update: function (o) {
+            if (typeof o === 'object') {
+                var difference;
+
+                // if using int, convert to percent to check difference
+                if (typeof o.arcFillInt !== 'undefined' && o.arcFillInt == this.arcFillInt &&
+                    typeof o.arcFillTotal !== 'undefined' && o.arcFillTotal == this.arcFillTotal) {
+                    o.arcFillPercent = this.arcFillPercent;
+                } else if (typeof o.arcFillInt !== 'undefined' && typeof o.arcFillTotal !== 'undefined' &&
+                    (o.arcFillInt != this.arcFillInt || o.arcFillTotal == this.arcFillTotal)) {
+                    o.arcFillPercent = (o.arcFillInt / o.arcFillTotal);
+                } else if (typeof o.arcFillInt !== 'undefined' && typeof o.arcFillTotal === 'undefined' &&
+                    (o.arcFillInt != this.arcFillInt)) {
+                    o.arcFillPercent = (o.arcFillInt / this.arcFillTotal);
+                }
+
+                if (typeof o.arcFillPercent !== 'undefined') {
+                    difference = Math.abs((this.arcFillPercent - o.arcFillPercent));
+                } else {
+                    difference = this.arcFillPercent;
+                }
+
+                this._extendOptions(o, true);
+
+                clearInterval(this._animateLoop);
+
+                if (difference > 0) {
+                    var that = this;
+                    this._animateLoop = setInterval(function () {
+                        return that._animate();
+                    }, (this.animateSpeed * this.animateNumerator) / (difference * this.animateDivisor));
+                }
+            }
+        },
+
+        _extendOptions: function (o, update) {
+            var color = false;
+            if (update)
+                color = this.colorArcFg;
+
+            $.extend(this, o, true);
+
+            if (typeof o.arcFillStart !== 'undefined' && typeof o.arcFillEnd !== 'undefined' && typeof o.arcFillTotal !== 'undefined') {
+                this.arcFillInt = (o.arcFillEnd - o.arcFillStart);
+            }
+
+            if (typeof o.arcFillPercent === 'undefined' && this.arcFillInt !== null && this.arcFillInt >= 0 && this.arcFillTotal !== null && this.arcFillTotal > 0) {
+                this.arcFillPercent = this.arcFillInt / this.arcFillTotal;
+            }
+
+            if (typeof o.elementId === 'undefined') {
+                this.elementId = 'fg-' + this.appendTo + '-canvas';
+            }
+            // supporting color if pass, changing to hex
+            if (typeof o.colorArcFg !== 'undefined') {
+                this.colorArcFg = colorToHex(o.colorArcFg);
+            }
+
+            if (typeof o.colorArcBg !== 'undefined') {
+                this.colorArcBg = colorToHex(o.colorArcBg);
+            }
+
+            // only use the styleArcFg if colorArcFg wasn't specified in the options
+            if (typeof o.styleArcFg !== 'undefined' && typeof o.colorArcFg === 'undefined') {
+                this.colorArcFg = getStyleRuleValue(this.styleSrc, this.styleArcFg);
+            }
+
+            if (typeof o.colorArcBg === 'undefined' && this.colorArcBg === null && this.colorArcFg !== null) {
+                this.colorArcBg = this.colorArcFg;
+            }
+
+            if (typeof this.colorArcBg !== null && (!update || colorToHex(this.colorArcFg) != colorToHex(color))) {
+                if (colorToHex(this.colorArcFg) != colorToHex(color))
+                    this.colorArcBg = this.colorArcFg;
+
+                this.colorArcBg = shadeColor(this.colorArcBg, this.arcBgColorLight, this.arcBgColorSat);
+            }
+
+            if (typeof o.dialLabel === 'boolean' && o.dialLabel) {
+                this.dialLabel = 'FlexGauge';
+            }
+
+        },
+
+        _build: function () {
+            if (document.getElementById(this.elementId) === null) {
+                $(this.appendTo).append('<canvas id="' + this.elementId + '" width="' + this.elementWidth + '" height="' + this.elementHeight + '"></canvas>');
+            }
+
+            this._canvas = document.getElementById(this.elementId);
+            this._ctx = this._canvas.getContext("2d");
+
+            this.arcAngleStart = this.arcAngleStart * Math.PI;
+            this.arcAngleEnd = this.arcAngleEnd * Math.PI;
+            if (this.animateEasing === false) {
+                this._animatePerc = this.arcFillPercent;
+            }
+
+            var that = this;
+            this._animateLoop = setInterval(function () {
+                return that._animate();
+            }, (this.animateSpeed * this.animateNumerator) / (this.arcFillPercent * this.animateDivisor));
+        },
+
+        _animate: function () {
+            var animateInt = Math.round(this._animatePerc * 100);
+            var arcInt = Math.round(this.arcFillPercent * 100);
+
+            if (animateInt < arcInt)
+                animateInt++;
+            else
+                animateInt--;
+
+            this._animatePerc = (animateInt / 100);
+            if (animateInt === arcInt) {
+                this.arcFillPercent = this._animatePerc;
+                clearInterval(this._animateLoop);
+                this._draw();
+            }
+            this._draw();
+        },
+
+        _draw: function () {
+            //Clear the canvas everytime a chart is drawn
+            this._ctx.clearRect(0, 0, this.elementWidth, this.elementHeight);
+
+            //Background 360 degree arc
+            this._ctx.beginPath();
+            this._ctx.strokeStyle = this.colorArcBg;
+            this._ctx.lineWidth = this.arcStrokeBg;
+            this._ctx.arc(
+                this.elementWidth / 2,
+                this.elementHeight / 2 + 50,
+                this.arcSize,
+                0,
+                Math.PI,
+                true
+            );
+
+            this._ctx.stroke();
+
+            //var newEnd = ((this.arcAngleEnd - this.arcAngleStart) * this._animatePerc) + this.arcAngleStart;
+            var newStart;
+            var newEnd;
+
+            var incArc = this.inc*Math.PI/2;
+            if (this.inc >= 0.0){
+                newStart = -Math.PI/2;
+                newEnd = newStart + incArc;
+            } else {
+                newStart = -Math.PI/2 + incArc;
+                newEnd = -Math.PI/2;
+            }
+
+            var colorShadesTabRed = ['#ff0000','#ff4000','#ff8000','#ff9900','#ffbf00','#ffff00'];
+            var colorShadesTabGreen = ['#ffff00','#E0FF00','#D0FF00','#a0ff00','#00ff00','#00ff40',];
+            var colorValue = parseInt(Math.abs((this.inc / this.incTot) * 5));
+            var theColor;
+            if (this.inc >= 0.0)
+                theColor = colorShadesTabGreen[colorValue];
+            else
+                theColor = colorShadesTabRed[5-colorValue];
+            this.colorArcFg = theColor;
+
+            this._ctx.beginPath();
+            this._ctx.strokeStyle = this.colorArcFg;
+            this._ctx.lineWidth = this.arcStrokeFg;
+            this._ctx.arc(
+                this.elementWidth / 2,
+                this.elementHeight / 2 + 50,
+                this.arcSize,
+                newStart,
+                newEnd,
+                false
+            );
+            this._ctx.stroke();
+            this._renderLabel();
+        },
+
+        _renderLabel: function () {
+            if (this.dialValue) {
+                var dialVal;
+                var dial = $(this.appendTo).find('div.' + this.dialClass);
+                if (dial.length === 0) {
+                    $(this.appendTo).append('<div class="' + this.dialClass + '"></div>');
+                }
+                dial = $(this.appendTo).find('div.' + this.dialClass);
+                if (typeof this.dialValue === 'boolean') {
+                    switch (this.dialUnit) {
+                        case '%':
+                            dialVal = Math.round(this._animatePerc * 100);
+                            break;
+                        default:
+                            dialVal = Math.round(this.arcFillInt * (this._animatePerc / this.arcFillPercent));
+                            break;
+                    }
+                    dialVal = (isNaN(dialVal) ? 0 : dialVal);
+                    switch (this.dialUnitPosition) {
+                        case 'before':
+                            dialVal = this.dialUnit + dialVal;
+                            break;
+                        case 'after':
+                            dialVal = dialVal + this.dialUnit;
+                            break;
+                    }
+                } else {
+                    dialVal = this.dialValue;
+                }
+                dial.html(dialVal)
+            }
+            if (this.dialLabel) {
+                var label = $(this.appendTo).find('div.' + this.dialLabelClass);
+                if (label.length === 0) {
+                    $(this.appendTo).append('<div class="' + this.dialLabelClass + '"></div>');
+                }
+                label = $(this.appendTo).find('div.' + this.dialLabelClass);
+                label.html(this.dialLabel);
+            }
+        }
+    };
+
+    function shadeColor(col, amt, sat) {
+        if (col[0] == "#") {
+            col = col.slice(1);
+        }
+
+        var num = parseInt(col, 16);
+
+        var r = (num >> 16) + amt;
+
+        if (r > 255) r = 255;
+        else if (r < 0) r = 0;
+
+        var b = ((num >> 8) & 0x00FF) + amt;
+
+        if (b > 255) b = 255;
+        else if (b < 0) b = 0;
+
+        var g = (num & 0x0000FF) + amt;
+
+        if (g > 255) g = 255;
+        else if (g < 0) g = 0;
+
+        var gray = r * 0.3086 + g * 0.6094 + b * 0.0820;
+        sat = (sat / 100);
+
+        r = Math.round(r * sat + gray * (1 - sat));
+        g = Math.round(g * sat + gray * (1 - sat));
+        b = Math.round(b * sat + gray * (1 - sat));
+        return "#" + (g | (b << 8) | (r << 16)).toString(16);
+    }
+
+    function getStyleRuleValue(style, selector) {
+        $('body').append('<div id="getStyleRuleValue-' + selector + '"></div>');
+        var element = $('#getStyleRuleValue-' + selector);
+        element.addClass(selector);
+        var color = element.css(style);
+        var hex = colorToHex(color);
+        element.remove();
+        return hex;
+    }
+
+    function colorToHex(color) {
+        if (color[0] != 'r')
+            return color;
+
+        var rgb = color.match(/^rgb\((\d+),\s*(\d+),\s*(\d+)\)$/);
+        return "#" +
+            ("0" + parseInt(rgb[1], 10).toString(16)).slice(-2) +
+            ("0" + parseInt(rgb[2], 10).toString(16)).slice(-2) +
+            ("0" + parseInt(rgb[3], 10).toString(16)).slice(-2);
+    }
+
+    if (typeof define === 'function') {
+        define('flex-gauge', ['jquery'], function ($) {
+            return FlexGauge;
+        });
+    } else {
+        window.FlexGauge = FlexGauge;
+    }
+})(jQuery);

var/www/static/js/indexjavascript.js

@@ -309,7 +309,13 @@ $(document).ready(function () {
                     var tmp_values2 = [];
                     refresh();
                     update_values();
-                    create_queue_table();
+
+                    if($('#button-toggle-queues').prop('checked')){
+                        create_queue_table();
+                    }
+                    else{
+                        $("#queueing").html('');
+                    }
 
 
                     for (i = 0; i < (glob_tabvar.row1).length; i++){

var/www/static/js/sentiment_plot.js

@@ -0,0 +1,94 @@
+var li_text = "<li><div class='checkbox'></div><label class='provider'><input value='" 
+var li_text_mid = "' type='checkbox'></input> "
+var li_text_end = "</label></li>"
+
+
+/* Get Providers List and display them by row */
+$.getJSON('/sentiment_analysis_plot_tool_getdata/?getProviders=True', function(data){
+    for(i=0; i<data.length; i++){
+        var providerList = i%2 == 0 ? '#providerList1' : '#providerList2';
+        $(providerList).append(li_text + data[i] + li_text_mid + data[i] + li_text_end);
+    }
+});
+
+
+/* Create the slider and button*/
+var today = Date.now();
+var old_day = today - (31*24*60*60)*1000;
+$( ".sliderRange" ).slider({
+    range: true,
+    min: old_day,
+    max: today,
+    values: [ today - (7*24*60*60)*1000, today ],
+    step: 24*60*60*1000,
+    slide: function( event, ui ) {
+        $( "#amount" ).val( new Date(ui.values[ 0 ]).toLocaleDateString() + " - " + new Date(ui.values[ 1 ]).toLocaleDateString() );
+    }
+});
+
+$( "#amount" ).val( new Date($( ".sliderRange" ).slider( "values", 0 )).toLocaleDateString() +
+  " - " + new Date($( ".sliderRange" ).slider( "values", 1 )).toLocaleDateString() );
+
+$('#plot_btn').click(plotData); 
+
+/* Plot the requested data (if available) stored in slider and checkboxes */
+function plotData(){
+
+    var graph_options = {
+                            series: {
+            	                stack: $('#checkbox_stacked').prop('checked'),
+                                lines: { show: false,
+                                lineWidth: 2,
+                                fill: true, fillColor: { colors: [ { opacity: 0.5 }, { opacity: 0.2 } ] }
+                                },
+                                bars: {show: true, barWidth: 60*60*1000},
+                                shadowSize: 0
+                            },
+                            grid: { 
+                                hoverable: true, 
+                                clickable: true, 
+                                tickColor: "#f9f9f9",
+                                borderWidth: 0
+                            },
+                            xaxis: {
+                                  mode: "time",
+                                  timeformat: "%m/%d:%Hh",
+                                  minTickSize: [1, "hour"]
+                            },
+                            yaxis: {
+                                autoscaleMargin: 0.1,
+                            },
+                        }
+    var query = $( "input:checked[value]" ).map(function () {return this.value;}).get().join(",");
+    var Qdate = new Date($( ".sliderRange" ).slider( "values", 0 )).toLocaleDateString() +'-'+ new Date($( ".sliderRange" ).slider( "values", 1 )).toLocaleDateString();
+        
+
+    // retreive the data from the server
+    $.getJSON('/sentiment_analysis_plot_tool_getdata/?getProviders=False&query='+query+'&Qdate='+Qdate, function(data){
+        var to_plot = [];
+        for (provider in data){
+            var nltk_data = Object.keys(data[provider]).map(function (key) { return data[provider][key]; });
+            var nltk_key = Object.keys(data[provider]).map(function (key) { return key; });
+
+            var pos = 0.0;
+            var neg = 0.0;
+            var XY_data = [];
+            for (i=0; i<nltk_data.length; i++){
+                if (nltk_data[i].length == 0)
+                    continue;
+                else {
+                    for (j=0; j<nltk_data[i].length; j++){
+                        var curr_data = jQuery.parseJSON(nltk_data[i][j].replace(/\'/g, '\"'));
+                        pos += curr_data['pos'];
+                        neg += curr_data['neg'];
+                    }
+                    pos = pos/nltk_data.length;
+                    neg = neg/nltk_data.length;
+                }
+                XY_data.push([nltk_key[i]*1000, pos-neg]);
+            }
+        to_plot.push({ data: XY_data, label: provider}); 
+        }
+	var plot = $.plot($("#graph"), to_plot, graph_options);
+    });
+}

var/www/static/js/sentiment_trending.js

@@ -0,0 +1,386 @@
+
+ function generate_offset_to_time(num){
+     var to_ret = {};
+     for(i=0; i<=num; i++) {
+         var t1 = new Date().getHours()-(23-i);
+         t1 = t1 < 0 ? 24+t1 : t1;
+         to_ret[i] =  t1+'h';
+     }
+     return to_ret;
+ };
+
+ function generate_offset_to_date(day){
+     day = day-1;
+     var now = new Date();
+     var to_ret = {};
+     for(i=day; i>=0; i--){
+         for(j=0; j<24; j++){
+             var t1 =now.getDate()-i + ":"; 
+             var t2 =now.getHours()-(23-j);
+             t2 = t2 < 0 ? 24+t2 : t2;
+             t2 += "h";
+             to_ret[j+24*(day-i)] = t1+t2;
+         }
+     }
+     return to_ret;
+ };
+
+ var offset_to_time = generate_offset_to_time(23);
+ var offset_to_date = generate_offset_to_date(7);
+
+ var sparklineOptions = {
+        height: 80,//Height of the chart - Defaults to 'auto' (line height of the containing tag)
+
+        chartRangeMin: -1,
+        chartRangeMax: 1,
+
+        type: 'bar',
+        barSpacing: 0,
+        barWidth: 2,
+        barColor: '#00bf5f',
+        negBarColor: '#f22929',
+        zeroColor: '#ffff00',
+            
+        tooltipFormat: '<span style="color: {{color}}">&#9679;</span> {{offset:names}}, {{value}} </span>',
+};
+
+
+$.getJSON("/sentiment_analysis_getplotdata/",
+    function(data) {
+        var all_data = [];
+        var plot_data = [];
+        var graph_avg = [];
+        var array_provider = Object.keys(data);
+        var dates_providers = Object.keys(data[array_provider[0]]);
+        var dateStart = parseInt(dates_providers[0]);
+        var oneHour = 60*60;
+        var oneWeek = oneHour*24*7;
+
+        var all_graph_day_sum = 0.0;
+        var all_graph_hour_sum = 0.0;
+        var all_graph_hour_maxVal = 0.0;
+        var all_day_avg = 0.0;
+        var all_day_avg_maxVal = 0.0;
+
+        for (graphNum=0; graphNum<8; graphNum++) {
+            var max_value = 0.0;
+            var max_value_day = 0.0;
+            var graph_data = [];
+            var spark_data = [];
+            var curr_provider = array_provider[graphNum];
+            var curr_sum = 0.0;
+            var curr_sum_elem = 0.0;
+            var day_sum = 0.0;
+            var day_sum_elem = 0.0;
+            var hour_sum = 0.0;
+
+            for(curr_date=dateStart+oneHour; curr_date<=dateStart+oneWeek; curr_date+=oneHour){
+                var data_array = data[curr_provider][curr_date];
+
+                if (data_array.length == 0){
+                    graph_data.push({'neg': 0.0, 'neu': 0.0, 'pos': 0.0, 'compoundPos': 0.0, 'compoundNeg': 0.0});
+                    spark_data.push(0);
+                } else { //compute avg for a given date for a given graph
+                    var compPosAvg = 0;
+                    var compNegAvg = 0;
+                    var pos = 0;
+                    var neg = 0;
+                    var neu = 0;
+
+                    for(i=0; i<data_array.length; i++){
+                        //console.log(data_array[i].replace(/\'/g, '\"'));
+                        var curr_data = jQuery.parseJSON(data_array[i].replace(/\'/g, '\"'));
+                        compPosAvg += curr_data['compoundPos'];
+                        compNegAvg += curr_data['compoundNeg'];
+                        pos += curr_data['pos'];
+                        neg += curr_data['neg'];
+                        neu += curr_data['neu'];
+                    }
+                    compPosAvg = compPosAvg/data_array.length;
+                    compNegAvg = compNegAvg/data_array.length;
+                    pos = pos/data_array.length;
+                    neg = neg/data_array.length;
+                    neu = neu/data_array.length;
+
+                    graph_data.push({'neg': neg, 'neu': neu, 'pos': pos, 'compoundPos': compPosAvg, 'compoundNeg': compNegAvg});
+                    spark_data.push(pos-neg);
+                    curr_sum += (pos-neg);
+                    curr_sum_elem++;
+                    max_value = Math.abs(pos-neg) > max_value ? Math.abs(pos-neg) : max_value;
+
+                    if(curr_date >= dateStart+oneWeek-23*oneHour){
+                        max_value_day = Math.abs(pos-neg) > max_value_day ? Math.abs(pos-neg) : max_value_day;
+                        day_sum += (pos-neg);
+                        day_sum_elem++;
+                    }
+                    if(curr_date > dateStart+oneWeek-2*oneHour && curr_date <=dateStart+oneWeek-oneHour){
+                        hour_sum += (pos-neg);
+                    }
+
+                }
+            }
+            all_graph_day_sum += day_sum;
+            all_graph_hour_sum += hour_sum;
+            all_graph_hour_maxVal = Math.abs(hour_sum) > all_graph_hour_maxVal ? Math.abs(hour_sum) : all_graph_hour_maxVal;
+
+            var curr_avg = curr_sum / (curr_sum_elem); 
+            if(isNaN(curr_avg))
+                curr_avg = 0.0
+            //var curr_avg = curr_sum / (oneWeek/oneHour); 
+            //var curr_avg = curr_sum / (spark_data.length); 
+            graph_avg.push([curr_provider, curr_avg]);
+            plot_data.push(spark_data);
+            all_data.push(graph_data);
+        
+
+            sparklineOptions.chartRangeMax = max_value;
+            sparklineOptions.chartRangeMin = -max_value;
+            sparklineOptions.tooltipValueLookups = { names: offset_to_date};
+
+            // print week
+            var num = graphNum + 1;
+            var placeholder = '.sparkLineStatsWeek' + num;
+            sparklineOptions.barWidth = 2;
+            $(placeholder).sparkline(plot_data[graphNum], sparklineOptions);
+            $(placeholder+'t').text(curr_provider);
+            var curr_avg_text = isNaN(curr_avg) ? "No data" : curr_avg.toFixed(5); 
+            $(placeholder+'s').text(curr_avg_text);
+        
+            sparklineOptions.barWidth = 18;
+            sparklineOptions.tooltipFormat = '<span style="color: {{color}}">&#9679;</span> Avg: {{value}} </span>'
+            $(placeholder+'b').sparkline([curr_avg], sparklineOptions);
+            sparklineOptions.tooltipFormat = '<span style="color: {{color}}">&#9679;</span> {{offset:names}}, {{value}} </span>'
+
+            sparklineOptions.tooltipValueLookups = { names: offset_to_time};
+            sparklineOptions.chartRangeMax = max_value_day;
+            sparklineOptions.chartRangeMin = -max_value_day;
+
+            var avgName = ".pannelWeek" + num;
+            if (curr_avg > 0) {
+                $(avgName).addClass("panel-success")
+            } else if(curr_avg < 0) {
+                $(avgName).addClass("panel-danger")
+            } else if(isNaN(curr_avg)) {
+                $(avgName).addClass("panel-info")
+            } else {
+                $(avgName).addClass("panel-warning")
+            }
+
+
+
+            // print today
+            var data_length = plot_data[graphNum].length;
+            var data_today = plot_data[graphNum].slice(data_length-24, data_length);
+
+            placeholder = '.sparkLineStatsToday' + num;
+            sparklineOptions.barWidth = 14;
+            $(placeholder).sparkline(data_today, sparklineOptions);
+            $(placeholder+'t').text(curr_provider);
+
+            sparklineOptions.barWidth = 18;
+            sparklineOptions.tooltipFormat = '<span style="color: {{color}}">&#9679;</span> Avg: {{value}} </span>'
+            //var day_avg = day_sum/24;
+            var day_avg = isNaN(day_sum/day_sum_elem) ? 0 : day_sum/day_sum_elem;
+            var day_avg_text = isNaN(day_sum/day_sum_elem) ? 'No data' : (day_avg).toFixed(5);
+            all_day_avg += day_avg;
+            all_day_avg_maxVal = Math.abs(day_avg) > all_day_avg_maxVal ? Math.abs(day_avg) : all_day_avg_maxVal;
+            $(placeholder+'b').sparkline([day_avg], sparklineOptions);
+            sparklineOptions.tooltipFormat = '<span style="color: {{color}}">&#9679;</span> {{offset:names}}, {{value}} </span>'
+            $(placeholder+'s').text(day_avg_text);
+
+            avgName = ".pannelToday" + num;
+            if (day_avg > 0) {
+                $(avgName).addClass("panel-success")
+            } else if(day_avg < 0) {
+                $(avgName).addClass("panel-danger")
+            } else if(isNaN(day_sum/day_sum_elem)) {
+                $(avgName).addClass("panel-info")
+            } else {
+                $(avgName).addClass("panel-warning")
+            }
+
+        }//for loop
+
+
+
+        /* ---------------- Gauge ---------------- */
+        var gaugeOptions = {
+            animateEasing: true,
+        
+            elementWidth: 200,
+            elementHeight: 125,
+        
+            arcFillStart: 10,
+            arcFillEnd: 12,
+            arcFillTotal: 20,
+            incTot: 1.0,
+        
+            arcBgColorLight: 200,
+            arcBgColorSat: 0,
+            arcStrokeFg: 20,
+            arcStrokeBg: 30,
+        
+            colorArcFg: '#FF3300',
+            animateSpeed: 1,
+        
+        };
+        // Clone object
+        var gaugeOptions2 = jQuery.extend(true, {}, gaugeOptions);
+        var gaugeOptions3 = jQuery.extend(true, {}, gaugeOptions);
+        
+        
+        
+        gaugeOptions.appendTo = '#gauge_today_last_hour';
+        gaugeOptions.dialLabel = 'Last hour';
+        gaugeOptions.elementId = 'gauge1';
+        var piePercent = (all_graph_hour_sum / 8) / all_graph_hour_maxVal;
+        gaugeOptions.inc = piePercent;
+        var gauge_today_last_hour = new FlexGauge(gaugeOptions);
+        
+        gaugeOptions2.appendTo = '#gauge_today_last_days';
+        gaugeOptions2.dialLabel = 'Today';
+        gaugeOptions2.elementId = 'gauge2';
+        //piePercent = (all_graph_day_sum / (8*24)) / max_value;
+        piePercent = (all_day_avg / 8) / all_day_avg_maxVal;
+        gaugeOptions2.inc = piePercent;
+        var gauge_today_last_days = new FlexGauge(gaugeOptions2);
+        
+        gaugeOptions3.appendTo = '#gauge_week';
+        gaugeOptions3.dialLabel = 'Week';
+        gaugeOptions3.elementId = 'gauge3';
+
+        var graph_avg_sum = 0.0;
+        var temp_max_val = 0.0;
+        for (i=0; i<graph_avg.length; i++){
+            graph_avg_sum += graph_avg[i][1];
+            temp_max_val = Math.abs(graph_avg[i][1]) > temp_max_val ? Math.abs(graph_avg[i][1]) : temp_max_val;
+        }
+
+        piePercent = (graph_avg_sum / graph_avg.length) / temp_max_val;
+        gaugeOptions3.inc = piePercent;
+        var gauge_today_last_days = new FlexGauge(gaugeOptions3);
+
+
+        /* --------- Sort providers -------- */
+
+        graph_avg.sort(function(a, b){return b[1]-a[1]});
+
+        for (i=1; i<6; i++){
+            $('.worst'+i).text(graph_avg[7-(i-1)][0]);
+            $('.best'+i).text(graph_avg[i-1][0]);
+        }
+
+        /* ----------- CanvasJS ------------ */
+
+        var comp_sum_day_pos = 0.0;
+        var comp_sum_day_neg = 0.0;
+        var comp_sum_hour_pos = 0.0;
+        var comp_sum_hour_neg = 0.0;
+        for(graphNum=0; graphNum<8; graphNum++){
+            curr_graphData = all_data[graphNum];
+            var gauge_data = curr_graphData.slice(curr_graphData.length-24, curr_graphData.length);
+            for (i=1; i< gauge_data.length; i++){
+                comp_sum_day_pos += gauge_data[i].compoundPos;
+                comp_sum_day_neg += gauge_data[i].compoundNeg;
+
+                if(i == 23){
+                    comp_sum_hour_pos += gauge_data[i].compoundPos;
+                    comp_sum_hour_neg += gauge_data[i].compoundNeg;
+                }
+            }
+
+        }
+
+        var options_canvasJS_1 = {
+          
+            animationEnabled: true,
+            axisY: {
+                tickThickness: 0,
+                lineThickness: 0,
+                valueFormatString: " ",
+                gridThickness: 0              
+            },
+            axisX: {
+                tickThickness: 0,
+                lineThickness: 0,
+                labelFontSize: 0.1,
+            },
+            data: [
+                {
+                    toolTipContent: "<span style='\"'color: {color};'\"'><strong>Positive: </strong></span><span><strong>{y}</strong></span>",
+                    type: "bar",
+                    color: "green",
+                    dataPoints: [
+                        {y: comp_sum_hour_pos/8}
+                    ]
+                },
+                {
+                    toolTipContent: "<span style='\"'color: {color};'\"'><strong>Negative: </strong></span><span><strong>{y}</strong></span>",
+                    type: "bar",
+                    color: "red",
+                    dataPoints: [
+                        {y: comp_sum_hour_neg/8}
+                    ]
+                }
+            ]
+        };
+        
+        var chart_canvas1 = new CanvasJS.Chart("bar_today_last_hour", options_canvasJS_1);
+
+        var options_canvasJS_2 = {
+          
+            animationEnabled: true,
+            axisY: {
+                tickThickness: 0,
+                lineThickness: 0,
+                valueFormatString: " ",
+                gridThickness: 0              
+            },
+            axisX: {
+                tickThickness: 0,
+                lineThickness: 0,
+                labelFontSize: 0.1,
+            },
+            data: [
+            {
+                toolTipContent: "<span style='\"'color: {color};'\"'><strong>Positive: </strong></span><span><strong>{y}</strong></span>",
+                type: "bar",
+                color: "green",
+                dataPoints: [
+                    {y: comp_sum_day_pos/8}
+                ]
+            },
+            {
+                toolTipContent: "<span style='\"'color: {color};'\"'><strong>Negative: </strong></span><span><strong>{y}</strong></span>",
+                type: "bar",
+                color: "red",
+                dataPoints: [
+                    {y: comp_sum_day_neg/8}
+                ]
+            }
+            ]
+        };
+
+        var chart_canvas2 = new CanvasJS.Chart("bar_today_last_days", options_canvasJS_2);
+        
+        chart_canvas1.render();
+        chart_canvas2.render();
+
+
+
+    }
+);
+
+
+
+
+
+
+
+
+
+
+
+
+
+

var/www/templates/Moduletrending.html

@@ -24,14 +24,7 @@
 
 	    <div id="wrapper">
 	      <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
-		  <div class="navbar-header">
-		  <ul class="nav navbar-nav">
-		      <li><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li>
-                      <li><a href="{{ url_for('trending') }}"><i class="glyphicon glyphicon-stats"></i> Trending charts</a></li>
-                      <li class="active"><a href="{{ url_for('moduletrending') }}"><i class="glyphicon glyphicon-stats"></i> Modules statistics</a></li>
-                      <li><a href="{{ url_for('browseImportantPaste') }}"><i class="fa fa-search-plus "></i> Browse important pastes</a></li>
-                  </ul>
-          </div>
+              {% include 'header.html' %}
           <!-- /.navbar-top-links -->
           <div class="navbar-default sidebar" role="navigation">
               <div class="sidebar-collapse">
@@ -62,6 +55,9 @@
         <script src="{{ url_for('static', filename='js/moduleTrending.js') }}"></script>
         <script>
             $(document).ready(function(){
+                activePage = "page-modulestats"
+                $("#"+activePage).addClass("active");
+
                 $("[align]").css({padding: "2px", width: 'auto', 'background': "rgba(102, 102, 102, 0.15)" , 'border': "3px solid rgb(102, 102, 102)"})
 
                refreshPlot(true); 

var/www/templates/Trending.html

@@ -32,14 +32,7 @@
 
 	    <div id="wrapper">
 	      <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
-		  <div class="navbar-header">
-		  <ul class="nav navbar-nav">
-		      <li><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li>
-                      <li class="active"><a href="{{ url_for('trending') }}"><i class="glyphicon glyphicon-stats"></i> Trending charts</a></li>
-                      <li><a href="{{ url_for('moduletrending') }}"><i class="glyphicon glyphicon-stats"></i> Modules statistics</a></li>
-                      <li><a href="{{ url_for('browseImportantPaste') }}"><i class="fa fa-search-plus "></i> Browse important pastes</a></li>
-                  </ul>
-          </div>
+                  {% include 'header.html' %}
           <!-- /.navbar-top-links -->
           <div class="navbar-default sidebar" role="navigation">
               <div class="sidebar-collapse">
@@ -58,7 +51,7 @@
         <div id="page-wrapper">
           <div class="row">
               <div class="col-lg-12">
-                  <h1 class="page-header">Trending charts</h1>
+                  <h1 class="page-header" data-page="page-trendingchart">Trending charts</h1>
               </div>
               <!-- /.col-lg-12 -->
           </div>
@@ -150,6 +143,9 @@ $SCRIPT_ROOT = {{ request.script_root|tojson|safe }};
 
         <script>
             $(document).ready(function(){
+                activePage = $('h1.page-header').attr('data-page');
+                $("#"+activePage).addClass("active");
+
                 $("[align]").css({padding: "2px", width: 'auto', 'background': "rgba(102, 102, 102, 0.15)" , 'border': "3px solid rgb(102, 102, 102)"})
 
                 // Create the graph when the page has just loaded

var/www/templates/browse_important_paste.html

@@ -40,14 +40,7 @@
 
     <div id="wrapper">
       <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
-          <div class="navbar-header">
-              <ul class="nav navbar-nav">
-                  <li><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li>
-                  <li><a href="{{ url_for('trending') }}"><i class="glyphicon glyphicon-stats"></i> Trending charts</a></li>
-                  <li><a href="{{ url_for('moduletrending') }}"><i class="glyphicon glyphicon-stats"></i> Modules statistics</a></li>
-                  <li class="active"><a href="{{ url_for('browseImportantPaste') }}"><i class="fa fa-search-plus "></i> Browse important pastes</a></li>
-              </ul>
-          </div>
+      {% include 'header.html' %}
       <!-- /.navbar-top-links -->
           <div class="navbar-default sidebar" role="navigation">
               <div class="sidebar-collapse">
@@ -86,7 +79,7 @@
 <div id="page-wrapper">
   <div class="row">
       <div class="col-lg-12">
-          <h1 class="page-header">Browse important pastes</h1>
+          <h1 class="page-header" data-page="page-browse" >Browse important pastes</h1>
       </div>
       <!-- /.col-lg-12 -->
   </div>
@@ -136,6 +129,9 @@
 <!-- import graph function -->
 <script>
     $(document).ready(function(){
+        activePage = $('h1.page-header').attr('data-page');
+        $("#"+activePage).addClass("active");
+
         var dataPath = 'credential';
         $.get("{{ url_for('importantPasteByModule') }}"+"?moduleName="+dataPath, function(data, status){
             $('#'+dataPath+'-tab').html(data);

var/www/templates/header.html

@@ -0,0 +1,23 @@
+<div class="navbar-header">
+    <ul class="nav navbar-nav">
+        <li id='page-index'><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li>
+        <li id='page-trendingchart'><a href="{{ url_for('trending') }}"><i class="glyphicon glyphicon-stats"></i> Trending charts</a></li>
+        <li id='page-modulestats'><a href="{{ url_for('moduletrending') }}"><i class="glyphicon glyphicon-stats"></i> Modules statistics</a></li>
+        <li id='page-browse'><a href="{{ url_for('browseImportantPaste') }}"><i class="fa fa-search-plus "></i> Browse important pastes</a></li>
+        <li id='page-sentiment'><a class="dropdown-toggle" data-toggle="dropdown" href="#"><i class="fa fa-heart"></i> Sentiment Analysis
+            <span class="caret"></span></a>
+            <ul class="dropdown-menu">
+              <li><a href="{{ url_for('sentiment_analysis_trending') }}"><i class="fa fa-bar-chart-o"> </i> Sentiment trending</a></li>
+              <li><a href="{{ url_for('sentiment_analysis_plot_tool') }}"><i class="fa fa-wrench"> </i> Sentiment plot Tool</a></li>
+            </ul>
+        </li>
+        <li id='page-termsfrequency'><a class="dropdown-toggle" data-toggle="dropdown" href="#"><i class="fa fa-eye"></i> Terms frequency
+            <span class="caret"></span></a>
+            <ul class="dropdown-menu">
+              <li><a href="{{ url_for('terms_management') }}"><i class="fa fa-gear "> </i> Terms managements</a></li>
+              <li><a href="{{ url_for('terms_plot_top') }}"><i class="glyphicon glyphicon-fire"> </i> Terms plot top</a></li>
+              <li><a href="{{ url_for('terms_plot_tool') }}"><i class="fa fa-wrench"> </i> Terms plot tool</a></li>
+            </ul>
+        </li>
+    </ul>
+</div>

var/www/templates/important_paste_by_module.html

@@ -28,7 +28,7 @@
 <script>
 $(document).ready(function(){
     $('[data-toggle="tooltip"]').tooltip();
-    $('#myTable').dataTable();
+    $('#myTable').dataTable({ "order": [[ 2, "desc" ]] });
 });
 </script>
 
@@ -92,7 +92,8 @@ $(document).ready(function(){
        }
 
        if (final_index != start_index){ // still have data to display
-           $("#mymodalbody").find("#paste-holder").text($("#mymodalbody").find("#paste-holder").text()+complete_paste.substring(start_index+1, final_index+1)); // Append the new content
+           // Append the new content using text() and not append (XSS)
+           $("#mymodalbody").find("#paste-holder").text($("#mymodalbody").find("#paste-holder").text()+complete_paste.substring(start_index+1, final_index+1));
            start_index = final_index;
            if (flag_stop)
                nothing_to_display();
@@ -115,7 +116,6 @@ $(document).ready(function(){
    $('#myTable').on( 'draw.dt', function () {
        // On click, get html content from url and update the corresponding modal
        $("[data-toggle='modal']").unbind('click.openmodal').on("click.openmodal", function (event) {
-           console.log('hi');
            event.preventDefault();
            var modal=$(this);
            var url = " {{ url_for('showpreviewpaste')  }}?paste=" + $(this).attr('data-path') + "&num=" + $(this).attr('data-num');

var/www/templates/index.html

@@ -12,6 +12,7 @@
   <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
   <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
   <link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
+  <link href="{{ url_for('static', filename='css/switch_checkbox.css') }}" rel="stylesheet" type="text/css" />
   <!-- JS -->
   <script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
   <script src="{{ url_for('static', filename='js/jquery.js') }}"></script>
@@ -34,14 +35,7 @@
 
     <div id="wrapper">
       <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
-          <div class="navbar-header">
-           <ul class="nav navbar-nav">
-              <li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li>
-              <li><a href="{{ url_for('trending') }}"><i class="glyphicon glyphicon-stats"></i> Trending charts</a></li>
-              <li><a href="{{ url_for('moduletrending') }}"><i class="glyphicon glyphicon-stats"></i> Modules statistics</a></li>
-              <li><a href="{{ url_for('browseImportantPaste') }}"><i class="fa fa-search-plus "></i> Browse important pastes</a></li>
-           </ul>
-        </div>
+          {% include 'header.html' %}
           <!-- /.navbar-top-links -->
           <div class="navbar-default sidebar" role="navigation">
               <div class="sidebar-collapse">
@@ -64,6 +58,14 @@
                   </div>
               </div>
                   <!-- <div id="Graph_paste_num" style="height:90px; width:100%;"></div> -->
+              <div class='pull_right'>
+                  <label class="switch">
+                      <input id="button-toggle-queues" class="switch-input" type="checkbox" checked>
+                      <span class="switch-label" data-on="On" data-off="Off"></span>
+                      <span class="switch-handle"></span>
+                  </label>
+                  <strong style="top: 3px; position: relative;">Display queues</strong>
+              <div>
               <div class="table-responsive", id="queueing" style="margin-top:10px;"></div>
              <a  href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
           </div>
@@ -107,7 +109,7 @@
 		<option value="100">100</option>
 		</select>
                 </label>
-		<input id="checkbox_log_info" type="checkbox" value="info" checked="true"> INFO
+		<input id="checkbox_log_info" type="checkbox" value="info"> INFO
 		<input id="checkbox_log_warning" type="checkbox" value="warning" checked="true"> WARNING
 		<input id="checkbox_log_critical" type="checkbox" value="critical" checked="true"> CRITICAL
 
@@ -142,8 +144,13 @@
     </div>
     <script type="text/javascript" src="{{ url_for('static', filename='js/indexjavascript.js')}}"></script>
 
+    <script>
+        activePage = "page-index";
+        $("#"+activePage).addClass("active");
+    </script>
+
     <script type="text/javascript">
-    initfunc( "{{ url_for('static', filename='csv/wordstrendingdata.csv') }}", {{ request.script_root|tojson|safe }} );
+        initfunc( "{{ url_for('static', filename='csv/wordstrendingdata.csv') }}", {{ request.script_root|tojson|safe }} );
     </script>
 
     <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>

var/www/templates/search.html

@@ -23,12 +23,11 @@
       .tooltip-inner {
          text-align: left;
          height: 200%;
-         width: 200%;
          max-width: 500px;
          max-height: 500px;
          font-size: 13px;
       }
-      xmp {
+      pre {
          white-space:pre-wrap;
          word-wrap:break-word;
       }
@@ -42,12 +41,7 @@
 
     <div id="wrapper">
       <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
-          <div class="navbar-header">
-          <ul class="nav navbar-nav">
-               <li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li>
-               <li><a href="{{ url_for('trending') }}"><i class="glyphicon glyphicon-stats"></i> Trending charts</a></li>
-          </ul>
-          </div>
+          {% include 'header.html' %}
           <!-- /.navbar-top-links -->
           <div class="navbar-default sidebar" role="navigation">
               <div class="sidebar-collapse">
@@ -201,7 +195,8 @@
        }
 
        if (final_index != start_index){ // still have data to display
-           $("#mymodalbody").find("#paste-holder").append(complete_paste.substring(start_index+1, final_index+1)); // Append the new content
+           // Append the new content using text() and not append (XSS)
+           $("#mymodalbody").find("#paste-holder").text($("#mymodalbody").find("#paste-holder").text() + complete_paste.substring(start_index+1, final_index+1)); 
            start_index = final_index;
            if (flag_stop)
                nothing_to_display();

var/www/templates/sentiment_analysis_plot_tool.html

@@ -0,0 +1,150 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+  <title>Analysis Information Leak framework Dashboard</title>
+
+  <!-- Core CSS -->
+  <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
+  <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
+  <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
+  <link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
+  <link href="{{ url_for('static', filename='css/jquery-ui.min.css') }}" rel="stylesheet" type="text/css" />
+  <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+  <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/jquery-ui.min.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
+
+  <style>
+      .sparkLineStats ul {
+          padding-left:0;
+          list-style:none
+      }
+  </style>
+</head>
+<body>
+
+   <div id="wrapper">
+      <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
+      {% include 'header.html' %}
+      <!-- /.navbar-top-links -->
+          <div class="navbar-default sidebar" role="navigation">
+              <div class="sidebar-collapse">
+                  <ul class="nav" id="side-menu">
+                      <li class="sidebar-search">
+                          {% include 'searchbox.html' %}
+                      </li>
+                 </ul>
+                  <!-- /#side-menu -->
+              </div>
+              <!-- /.sidebar-collapse -->
+           <a  href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
+          </div>
+          <!-- /.navbar-static-side -->
+      </nav>
+   </div>
+
+<div id="page-wrapper">
+  <div class="row">
+      <div class="col-lg-12">
+          <h1 class="page-header" data-page="page-sentiment" >Sentiment analysis: Plot tool</h1>
+      </div>
+      <!-- /.col-lg-12 -->
+  </div>
+    <!-- /.row -->
+    <div class="row">
+
+    <!-- Panel OPTIONS -->
+    <div class="row">
+        <div class="col-lg-12">
+            <div class="row">
+                <div class="col-lg-12">
+                    <div id="panel-today" class="panel panel-default">
+                        <div class="panel-heading">
+                            <strong>Select options</strong>
+                        </div>
+                        <div class="panel-body">
+                            <div class="row">
+                            <!-- left column -->
+                            <div class="col-lg-9">
+                                <!-- providers charts -->
+                                <div class="col-lg-6">
+                                   <div class="sparkLineStats">
+                                       <ul id="providerList1">
+                                       </ul>
+                                   </div>
+                                </div>
+                                <div class="col-lg-6">
+                                   <div class="sparkLineStats">
+                                       <ul id="providerList2">
+                                       </ul>
+                                   </div>
+                                </div> 
+                            </div>
+                            <!-- right column -->
+                            <div class="col-lg-3">
+                                 <div aria-disabled="false" class="slider sliderRange sliderBlue ui-slider ui-slider-horizontal ui-widget ui-widget-content ui-corner-all" style="margin-bottom: 5px;"></div>
+                                 <strong>Date:</strong> <input type="text" id="amount" readonly style="border:0; color:#f6931f; font-weight:bold;">
+                            </div>
+                            </div>
+                            <!-- /.row -->
+                            <div class="row">
+                                <button id="plot_btn" class="btn btn-info" style="margin-right: 8px; margin-left: 8px;float: right;">Plot!</button>
+                                <label style="float: right;"><input id="checkbox_stacked" type="checkbox" checked=true> Stacked graph?</label>
+                            </div>
+                        </div>
+                        <!-- /.panel-body -->
+                    </div>
+                </div>
+                <!-- /.panel -->
+            </div>
+            <!-- /.panel -->
+        </div>
+    </div>
+    
+    <!-- Panel PLOT -->
+    <div class="row">
+        <div class="col-lg-12">
+            <div class="row">
+                <div class="col-lg-12">
+                    <div id="panel-today" class="panel panel-default">
+                        <div class="panel-heading">
+                            <strong>Graph</strong>
+                        </div>
+                        <div class="panel-body">
+                            <div id="graph" style="height: 300px;"></div>
+                        </div>
+                        <!-- /.panel-body -->
+                    </div>
+                </div>
+                <!-- /.panel -->
+            </div>
+            <!-- /.panel -->
+        </div>
+    </div>
+
+
+    <!-- /.row -->
+    </div>
+<!-- /#page-wrapper -->
+</div>
+
+
+<!-- import graph function -->
+<script>
+    $(document).ready(function(){
+        activePage = $('h1.page-header').attr('data-page');
+        $("#"+activePage).addClass("active");
+   });
+</script>
+<script src="{{ url_for('static', filename='js/sentiment_plot.js') }}"></script>
+
+</body>
+</html>

var/www/templates/sentiment_analysis_trending.html

@@ -0,0 +1,518 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+  <title>Analysis Information Leak framework Dashboard</title>
+
+  <!-- Core CSS -->
+  <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
+  <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
+  <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
+  <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+  <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/FlexGauge.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/jquery.sparkline.min.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/jquery.canvasjs.min.js') }}"></script>
+
+  <style>
+
+      .moodtable_worst {
+          background: rgba(255, 0, 0, 0.47);
+          font-size: small;
+      }
+
+      .moodtable_best {
+          background: rgba(132, 255, 0, 0.5);
+          font-size: small;
+      }
+
+      .jqstooltip{
+          box-sizing: content-box;
+      }
+
+      strong {
+          font-size: 16px;
+      }
+      
+      .table {
+          margin-bottom: 0px;
+      }
+
+      .sparkLineStats ul {
+          padding-left:0;
+          list-style:none
+      }
+      
+      .sparkLineStats {
+          position: relative;
+          margin-bottom: -4px;
+      }
+
+      .sparkLineStats ul li {
+          margin-bottom: 8px;
+          line-height: 90px;
+          font-size: 12px;
+      }
+
+      .sparkLineStats ul li div {
+          float: left;
+      }
+
+      .sparkLineStats ul li div:first-child {
+          margin-right: 5px;
+      }
+      
+      .panelInside {
+          padding: 5px;
+      }
+
+      .fg-dial-label {
+          font-size: 100%;
+          font-weight: bold;
+          left: 0;
+          position: relative;
+          text-align: center;
+          top: -60px;
+          margin-bottom: -10px;
+          width: 100%;
+      }
+  </style>
+
+</head>
+<body>
+
+   <div id="wrapper">
+      <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
+      {% include 'header.html' %}
+      <!-- /.navbar-top-links -->
+          <div class="navbar-default sidebar" role="navigation">
+              <div class="sidebar-collapse">
+                  <ul class="nav" id="side-menu">
+                      <li class="sidebar-search">
+                          {% include 'searchbox.html' %}
+                      </li>
+                 </ul>
+                  <!-- /#side-menu -->
+              </div>
+              <!-- /.sidebar-collapse -->
+           <a  href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
+          </div>
+          <!-- /.navbar-static-side -->
+      </nav>
+   </div>
+
+<div id="page-wrapper">
+  <div class="row">
+      <div class="col-lg-12">
+          <h1 class="page-header" data-page="page-sentiment" >Sentiment analysis: Trending</h1>
+      </div>
+      <!-- /.col-lg-12 -->
+  </div>
+    <!-- /.row -->
+    <div class="row">
+
+    <!-- Pannel TODAY -->
+    <div class="row">
+        <div class="col-lg-12">
+            <div class="row">
+                <div class="col-lg-12">
+                    <div id="panel-today" class="panel panel-default">
+                        <div class="panel-heading">
+                            <strong data-toggle="tooltip" data-placement="right" title="Providers displayed are in the top list in Module Statistics">Today's mood</strong>
+                        </div>
+                        <div class="panel-body">
+                            <!-- left column -->
+                            <div class="col-lg-9" style="padding-left: 0px;">
+                                <!-- providers charts -->
+                                <div class="col-lg-6">
+                                   <div class="sparkLineStats">
+                                       <div id="panel-today" class="panel panel-default pannelToday1">
+                                           <div class="panel-heading">
+                                               <strong class="sparkLineStatsToday1t">Graph 1</strong>
+                                               <strong class="sparkLineStatsToday1s pull-right">Avg</strong>
+                                           </div>
+                                           <div class="panel-body panelInside">
+                                               <table class="table">
+                                                   <tbody>
+                                                       <tr>
+                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday1"></div></td> 
+                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday1b"></div></td> 
+                                                       </tr>
+                                                    </tbody>
+                                               </table>
+                                           </div>
+                                       </div>
+                                       <div id="panel-today" class="panel panel-default pannelToday2">
+                                           <div class="panel-heading">
+                                               <strong class="sparkLineStatsToday2t">Graph 2</strong>
+                                               <strong class="sparkLineStatsToday2s pull-right">Avg</strong>
+                                           </div>
+                                           <div class="panel-body panelInside">
+                                               <table class="table">
+                                                   <tbody>
+                                                       <tr>
+                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday2"></div></td> 
+                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday2b"></div></td> 
+                                                       </tr>
+                                                    </tbody>
+                                               </table>
+                                           </div>
+                                       </div>
+                                       <div id="panel-today" class="panel panel-default pannelToday3">
+                                           <div class="panel-heading">
+                                               <strong class="sparkLineStatsToday3t">Graph 3</strong>
+                                               <strong class="sparkLineStatsToday3s pull-right">Avg</strong>
+                                           </div>
+                                           <div class="panel-body panelInside">
+                                               <table class="table">
+                                                   <tbody>
+                                                       <tr>
+                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday3"></div></td> 
+                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday3b"></div></td> 
+                                                       </tr>
+                                                    </tbody>
+                                               </table>
+                                           </div>
+                                        </div>
+                                       <div id="panel-today" class="panel panel-default pannelToday4">
+                                           <div class="panel-heading">
+                                               <strong class="sparkLineStatsToday4t">Graph 4</strong>
+                                               <strong class="sparkLineStatsToday4s pull-right">Avg</strong>
+                                           </div>
+                                           <div class="panel-body panelInside">
+                                               <table class="table">
+                                                   <tbody>
+                                                       <tr>
+                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday4"></div></td> 
+                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday4b"></div></td> 
+                                                       </tr>
+                                                    </tbody>
+                                               </table>
+                                           </div>
+                                        </div>
+                                   </div>
+                                </div>
+                                <div class="col-lg-6">
+                                   <div class="sparkLineStats">
+                                       <div id="panel-today" class="panel panel-default pannelToday5">
+                                           <div class="panel-heading">
+                                               <strong class="sparkLineStatsToday5t">Graph 5</strong>
+                                               <strong class="sparkLineStatsToday5s pull-right">Avg</strong>
+                                           </div>
+                                           <div class="panel-body panelInside">
+                                               <table class="table">
+                                                   <tbody>
+                                                       <tr>
+                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday5"></div></td> 
+                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday5b"></div></td> 
+                                                       </tr>
+                                                    </tbody>
+                                               </table>
+                                           </div>
+                                        </div>
+                                       <div id="panel-today" class="panel panel-default pannelToday6">
+                                           <div class="panel-heading">
+                                               <strong class="sparkLineStatsToday6t">Graph 6</strong>
+                                               <strong class="sparkLineStatsToday6s pull-right">Avg</strong>
+                                           </div>
+                                           <div class="panel-body panelInside">
+                                               <table class="table">
+                                                   <tbody>
+                                                       <tr>
+                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday6"></div></td> 
+                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday6b"></div></td> 
+                                                       </tr>
+                                                    </tbody>
+                                               </table>
+                                           </div>
+                                        </div>
+                                       <div id="panel-today" class="panel panel-default pannelToday7">
+                                           <div class="panel-heading">
+                                               <strong class="sparkLineStatsToday7t">Graph 7</strong>
+                                               <strong class="sparkLineStatsToday7s pull-right">Avg</strong>
+                                           </div>
+                                           <div class="panel-body panelInside">
+                                               <table class="table">
+                                                   <tbody>
+                                                       <tr>
+                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday7"></div></td> 
+                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday7b"></div></td> 
+                                                       </tr>
+                                                    </tbody>
+                                               </table>
+                                           </div>
+                                        </div>
+                                       <div id="panel-today" class="panel panel-default pannelToday8">
+                                           <div class="panel-heading">
+                                               <strong class="sparkLineStatsToday8t">Graph 8</strong>
+                                               <strong class="sparkLineStatsToday8s pull-right">Avg</strong>
+                                           </div>
+                                           <div class="panel-body panelInside">
+                                               <table class="table">
+                                                   <tbody>
+                                                       <tr>
+                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday8"></div></td> 
+                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsToday8b"></div></td> 
+                                                       </tr>
+                                                    </tbody>
+                                               </table>
+                                           </div>
+                                        </div>
+                                   </div>
+                                </div> 
+                            </div>
+                            <!-- right column -->
+                            <div class="col-lg-3">
+                                <div class="well text-center" style="padding: 0px;">
+                                    <strong data-toggle="tooltip" data-placement="top" title="Percentage is computed over the last hour max value">Mood value</strong>
+                                    <div id="gauge_today_last_hour"></div>
+                                    <strong data-toggle="tooltip" data-placement="top" title="Average of the sentiments' intensity">Compound by mood</strong>
+                                    <div id="bar_today_last_hour" style="height: 70px; width: 100%;"></div>
+                                </div>
+                                <div class="well text-center" style="padding: 0px;">
+                                    <strong data-toggle="tooltip" data-placement="top" title="Percentage is computed over the today max value">Mood value</strong>
+                                    <div id="gauge_today_last_days"></div>
+                                    <strong data-toggle="tooltip" data-placement="top" title="Average of the of sentiments' intensity">Compound by mood</strong>
+                                    <div id="bar_today_last_days" style="height: 70px; width: 100%;"></div>
+                                </div>
+                            </div>
+                            <!-- /.row -->
+                        </div>
+                        <!-- /.panel-body -->
+                    </div>
+                </div>
+                <!-- /.panel -->
+            </div>
+            <!-- /.panel -->
+        </div>
+    </div>
+
+    <!-- Pannel WEEK -->
+    <div class="row">
+        <div class="col-lg-12">
+            <div class="row">
+                <div class="col-lg-12">
+                    <div id="panel-week" class="panel panel-default">
+                        <div class="panel-heading">
+                            <strong>Week's mood</strong>
+                        </div>
+                        <div class="panel-body">
+                            <!-- left column -->
+                            <div class="col-lg-9" style="padding-left: 0px;">
+                                <!-- providers charts -->
+                                <div class="col-lg-6">
+                                   <div class="sparkLineStats">
+                                       <div id="panel-week" class="panel panel-default pannelWeek1">
+                                           <div class="panel-heading">
+                                               <strong class="sparkLineStatsWeek1t">Graph 1</strong>
+                                               <strong class="sparkLineStatsWeek1s pull-right">Avg</strong>
+                                           </div>
+                                           <div class="panel-body panelInside">
+                                               <table class="table">
+                                                   <tbody>
+                                                       <tr>
+                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek1"></div></td> 
+                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek1b"></div></td> 
+                                                       </tr>
+                                                    </tbody>
+                                               </table>
+                                           </div>
+                                       </div>
+                                       <div id="panel-week" class="panel panel-default pannelWeek2">
+                                           <div class="panel-heading">
+                                               <strong class="sparkLineStatsWeek2t">Graph 2</strong>
+                                               <strong class="sparkLineStatsWeek2s pull-right">Avg</strong>
+                                           </div>
+                                           <div class="panel-body panelInside">
+                                               <table class="table">
+                                                   <tbody>
+                                                       <tr>
+                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek2"></div></td> 
+                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek2b"></div></td> 
+                                                       </tr>
+                                                    </tbody>
+                                               </table>
+                                           </div>
+                                       </div>
+                                       <div id="panel-week" class="panel panel-default pannelWeek3">
+                                           <div class="panel-heading">
+                                               <strong class="sparkLineStatsWeek3t">Graph 3</strong>
+                                               <strong class="sparkLineStatsWeek3s pull-right">Avg</strong>
+                                           </div>
+                                           <div class="panel-body panelInside">
+                                               <table class="table">
+                                                   <tbody>
+                                                       <tr>
+                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek3"></div></td> 
+                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek3b"></div></td> 
+                                                       </tr>
+                                                    </tbody>
+                                               </table>
+                                           </div>
+                                       </div>
+                                       <div id="panel-week" class="panel panel-default pannelWeek4">
+                                           <div class="panel-heading">
+                                               <strong class="sparkLineStatsWeek4t">Graph 4</strong>
+                                               <strong class="sparkLineStatsWeek4s pull-right">Avg</strong>
+                                           </div>
+                                           <div class="panel-body panelInside">
+                                               <table class="table">
+                                                   <tbody>
+                                                       <tr>
+                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek4"></div></td> 
+                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek4b"></div></td> 
+                                                       </tr>
+                                                    </tbody>
+                                               </table>
+                                           </div>
+                                       </div>
+                                   </div>
+                                </div>
+                                <div class="col-lg-6">
+                                   <div class="sparkLineStats">
+                                       <div id="panel-week" class="panel panel-default pannelWeek5">
+                                           <div class="panel-heading">
+                                               <strong class="sparkLineStatsWeek5t">Graph 5</strong>
+                                               <strong class="sparkLineStatsWeek5s pull-right">Avg</strong>
+                                           </div>
+                                           <div class="panel-body panelInside">
+                                               <table class="table">
+                                                   <tbody>
+                                                       <tr>
+                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek5"></div></td> 
+                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek5b"></div></td> 
+                                                       </tr>
+                                                    </tbody>
+                                               </table>
+                                           </div>
+                                       </div>
+                                       <div id="panel-week" class="panel panel-default pannelWeek6">
+                                           <div class="panel-heading">
+                                               <strong class="sparkLineStatsWeek6t">Graph 6</strong>
+                                               <strong class="sparkLineStatsWeek6s pull-right">Avg</strong>
+                                           </div>
+                                           <div class="panel-body panelInside">
+                                               <table class="table">
+                                                   <tbody>
+                                                       <tr>
+                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek6"></div></td> 
+                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek6b"></div></td> 
+                                                       </tr>
+                                                    </tbody>
+                                               </table>
+                                           </div>
+                                       </div>
+                                       <div id="panel-week" class="panel panel-default pannelWeek7">
+                                           <div class="panel-heading">
+                                               <strong class="sparkLineStatsWeek7t">Graph 7</strong>
+                                               <strong class="sparkLineStatsWeek7s pull-right">Avg</strong>
+                                           </div>
+                                           <div class="panel-body panelInside">
+                                               <table class="table">
+                                                   <tbody>
+                                                       <tr>
+                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek7"></div></td> 
+                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek7b"></div></td> 
+                                                       </tr>
+                                                    </tbody>
+                                               </table>
+                                           </div>
+                                       </div>
+                                       <div id="panel-week" class="panel panel-default pannelWeek8">
+                                           <div class="panel-heading">
+                                               <strong class="sparkLineStatsWeek8t">Graph 8</strong>
+                                               <strong class="sparkLineStatsWeek8s pull-right">Avg</strong>
+                                           </div>
+                                           <div class="panel-body panelInside">
+                                               <table class="table">
+                                                   <tbody>
+                                                       <tr>
+                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek8"></div></td> 
+                                                           <td style="border-top: 0px solid #ddd;"><div class="sparkLineStatsWeek8b"></div></td> 
+                                                       </tr>
+                                                    </tbody>
+                                               </table>
+                                           </div>
+                                       </div>
+                                   </div>
+                                </div> 
+                            </div>
+                            <!-- right column -->
+                            <div class="col-lg-3">
+                                <div class="well text-center" style="padding: 0px;">
+                                    <strong data-toggle="tooltip" data-placement="top" title="Percentage is computed over the week max value">Mood value</strong>
+                                    <div id="gauge_week"></div>
+                                </div>
+                                <div class="well text-center" style="padding: 0px;">
+                                    <table class="table table-striped table-bordered table-hover" id="myTable">
+                                        <thead>
+                                             <tr>
+                                                <th>Worst mood</th>
+                                                <th>Best mood</th>
+                                             </tr>
+                                        </thead>
+                                        <tbody>
+                                          <tr>
+                                              <td class="worst1 moodtable_worst">worst1</td>
+                                              <td class="best1 moodtable_best">best1</td>
+                                          </tr>
+                                          <tr>
+                                              <td class="worst2 moodtable_worst">worst2</td>
+                                              <td class="best2 moodtable_best">best2</td>
+                                          </tr>
+                                          <tr>
+                                              <td class="worst3 moodtable_worst">worst3</td>
+                                              <td class="best3 moodtable_best">best3</td>
+                                          </tr>
+                                          <tr>
+                                              <td class="worst4 moodtable_worst">worst4</td>
+                                              <td class="best4 moodtable_best">best4</td>
+                                          </tr>
+                                          <tr>
+                                              <td class="worst5 moodtable_worst">worst5</td>
+                                              <td class="best5 moodtable_best">best5</td>
+                                          </tr>
+                                        </tbody>
+                                    </table>
+                                </div>
+                            </div>
+                            <!-- /.row -->
+                        </div>
+                        <!-- /.panel-body -->
+                    </div>
+                </div>
+                <!-- /.panel -->
+            </div>
+            <!-- /.panel -->
+        </div>
+    </div>
+
+
+
+    
+    <!-- /.row -->
+    </div>
+<!-- /#page-wrapper -->
+</div>
+
+
+<!-- import graph function -->
+<script src="{{ url_for('static', filename='js/sentiment_trending.js') }}"></script>
+<script>
+
+    $(document).ready(function(){
+        activePage = $('h1.page-header').attr('data-page');
+        $("#"+activePage).addClass("active");
+        $('[data-toggle="tooltip"]').tooltip(); 
+
+
+        // Reload every 30min
+        setTimeout(function(){ location.reload(); }, 30*60*1000);
+   });
+</script>
+
+</body>
+</html>

var/www/templates/terms_management.html

@@ -0,0 +1,351 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+  <title>Analysis Information Leak framework Dashboard</title>
+
+  <!-- Core CSS -->
+  <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
+  <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
+  <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
+  <link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
+  <link href="{{ url_for('static', filename='css/jquery-ui.min.css') }}" rel="stylesheet" type="text/css" />
+  <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+  <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
+
+  <style>
+      .sparkLineStats ul {
+          padding-left:0;
+          list-style:none
+      }
+
+      .btn-link {
+          color: #000000
+      }
+
+      .popover-content {
+         white-space:pre-wrap;
+         word-wrap:break-word;
+      }
+  </style>
+</head>
+<body>
+
+    <!-- Modal -->
+    <div id="mymodal" class="modal fade" role="dialog">
+      <div class="modal-dialog modal-lg">
+    
+        <!-- Modal content-->
+        <div id="mymodalcontent" class="modal-content">
+          <div id="mymodalbody" class="modal-body" max-width="8500px">
+            <p>Loading paste information...</p>
+            <img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" height="26" width="26" style="margin: 4px;">
+          </div>
+          <div class="modal-footer">
+            <a id="button_show_plot" target="_blank" href=""><button type="button" class="btn btn-info">Plot term</button></a>
+            <button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
+          </div>
+        </div>
+      </div>
+    </div>
+
+   <div id="wrapper">
+      <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
+      {% include 'header.html' %}
+      <!-- /.navbar-top-links -->
+          <div class="navbar-default sidebar" role="navigation">
+              <div class="sidebar-collapse">
+                  <ul class="nav" id="side-menu">
+                      <li class="sidebar-search">
+                          {% include 'searchbox.html' %}
+                      </li>
+                 </ul>
+                  <!-- /#side-menu -->
+              </div>
+              <!-- /.sidebar-collapse -->
+           <a  href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
+          </div>
+          <!-- /.navbar-static-side -->
+      </nav>
+   </div>
+
+<div id="page-wrapper">
+  <div class="row">
+      <div class="col-lg-12">
+          <h1 class="page-header" data-page="page-termsfrequency" >Terms frequency: Management interface</h1>
+      </div>
+      <!-- /.col-lg-12 -->
+  </div>
+    <!-- /.row -->
+    <div class="row">
+
+    <!-- Panel OPTIONS -->
+    <div class="row">
+        <div class="col-lg-12">
+            <div class="row">
+                <div class="col-lg-12">
+                    <div id="panel-today" class="panel panel-success">
+                        <div class="panel-heading">
+                            <strong>Manage tracked terms</strong>
+                        </div>
+                        <div class="panel-body">
+
+                            <div class="form-group input-group" style="margin-bottom: 30px;">
+                                <span class="input-group-addon"><span class="fa fa-eye"></span></span>
+                                <input id="followTermInput" class="form-control" placeholder="Term to track" type="text" style="max-width: 400px;">
+                                <button id="followTermBtn" class="btn btn-success btn-interaction" style="margin-left: 10px;" data-section="followTerm" data-action="add"> Add term</button>
+                            </div>
+
+                            <table class="table table-striped table-bordered table-hover" id="myTable">
+                                <thead>
+                                     <tr>
+                                        <th style="max-width: 800px;">Term</th>
+                                        <th>Added date</th>
+                                        <th>Day occurence</th>
+                                        <th>Week occurence</th>
+                                        <th>Month occurence</th>
+                                        <th># Concerned pastes</th>
+                                        <th>Action</th>
+                                     </tr>
+                                </thead>
+                                <tbody>
+                                {% set i = 0 %}
+                                {% for term in track_list %}
+                                  <tr>
+                                      <td>{{ term }}</td>
+                                      <td>{{ track_list_values[i][3] }}</td>
+                                      <td>{{ track_list_values[i][0] }}</td>
+                                      <td>{{ track_list_values[i][1] }}</td>
+                                      <td>{{ track_list_values[i][2] }}</td>
+                                      <td>{{ track_list_num_of_paste[i] }}</td>
+                                      <td><p style="margin: 0px;">
+                                          <span data-toggle="modal" data-target="#mymodal" data-term="{{ term }}" ><button class="btn-link" data-toggle="tooltip" data-placement="right" title="Show concerned paste(s)"><span class="glyphicon glyphicon-info-sign"></span></button></span>
+                                          <button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ term }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
+                                      </p></td>
+                                  </tr>
+                                {% set i = i + 1 %}
+                                {% endfor %}
+                                </tbody>
+                            </table>
+                        <!-- /.panel-body -->
+                    </div>
+                </div>
+                <!-- /.panel -->
+            </div>
+            <!-- /.panel -->
+        </div>
+    </div>
+    
+     <!-- Panel OPTIONS -->
+        <div class="col-lg-12">
+            <div class="row">
+                <div class="col-lg-12">
+                    <div id="panel-today" class="panel panel-danger">
+                        <div class="panel-heading">
+                            <strong>Manage blacklisted terms</strong>
+                        </div>
+                        <div class="panel-body">
+
+                            <div class="form-group input-group" style="margin-bottom: 30px;">
+                                <span class="input-group-addon"><span class="fa fa-eye-slash "></span></span>
+                                <input id="blacklistTermInput" class="form-control" placeholder="Term to track" type="text" style="max-width: 400px;">
+                                <button id="blacklistTermBtn" class="btn btn-danger btn-interaction" style="margin-left: 10px;" data-section="blacklistTerm" data-action="add"> Black list a term</button>
+                            </div>
+
+                            <table class="table table-striped table-bordered table-hover" id="myTable2">
+                                <thead>
+                                     <tr>
+                                        <th style="max-width: 800px;">Termx</th>
+                                        <th>Added date</th>
+                                        <th>Action</th>
+                                     </tr>
+                                </thead>
+                                <tbody>
+                                {% set i = 0 %}
+                                {% for term in black_list %}
+                                  <tr>
+                                      <td>{{ black_list[i][0] }}</td>
+                                      <td>{{ black_list[i][1] }}</td>
+                                      <td><p style="margin: 0px;">
+                                          <button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="right" title="Remove this term" data-content="{{ black_list[i][0] }}" data-section="blacklistTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
+                                      </p></td>
+                                  </tr>
+                                {% set i = i + 1 %}
+                                {% endfor %}
+                                </tbody>
+                            </table>
+                        <!-- /.panel-body -->
+                    </div>
+                </div>
+                <!-- /.panel -->
+            </div>
+            <!-- /.panel -->
+        </div>
+    </div>   
+
+    <!-- /.row -->
+    </div>
+<!-- /#page-wrapper -->
+</div>
+
+
+<!-- import graph function -->
+<script>
+    $(document).ready(function(){
+        activePage = $('h1.page-header').attr('data-page');
+        $("#"+activePage).addClass("active");
+
+
+        $('[data-toggle="tooltip"]').tooltip();
+	table_track = $('#myTable').DataTable();
+	table_black = $('#myTable2').DataTable();
+
+        $("#followTermInput").keyup(function(event){
+            if(event.keyCode == 13){
+                $("#followTermBtn").click();
+                $("#followTermInput").val("");
+            }
+        });
+        
+        $("#blacklistTermInput").keyup(function(event){
+            if(event.keyCode == 13){
+                $("#blacklistTermBtn").click();
+                $("#blacklistTermInput").val("");
+            }
+        });
+
+        perform_binding();
+
+        // On click, get html content from url and update the corresponding modal
+        $("[data-toggle='modal']").on("click.openmodal", function (event) {
+            //console.log(data); 
+            event.preventDefault();
+            var the_modal=$(this);
+            var url = "{{ url_for('terms_management_query_paste')  }}?term=" + $(this).attr('data-term');
+            $.getJSON(url, function (data) {
+                if (data.length != 0) {
+                    var html_to_add = "";
+                    html_to_add += "<table class=\"table table-striped\">";
+                    html_to_add += "<thead>";
+                      html_to_add += "<tr>";
+                        html_to_add += "<th>Source</th>";
+                        html_to_add += "<th>Date</th>";
+                        html_to_add += "<th>Encoding</th>";
+                        html_to_add += "<th>Size (Kb)</th>";
+                        html_to_add += "<th>Mime</th>";
+                        html_to_add += "<th>(# lines, Max line length)</th>";
+                        html_to_add += "<th>Preview</th>";
+                      html_to_add += "</tr>";
+                    html_to_add += "</thead>";
+                    html_to_add += "<tbody>";
+                    for (i=0; i<data.length; i++) {
+                        curr_data = data[i];
+                            html_to_add += "<tr>";
+                                html_to_add += "<td>"+curr_data.source+"</td>";
+                                html_to_add += "<td>"+curr_data.date+"</td>";
+                                html_to_add += "<td>"+curr_data.encoding+"</td>";
+                                html_to_add += "<td>"+curr_data.size+"</td>";
+                                html_to_add += "<td>"+curr_data.mime+"</td>";
+                                html_to_add += "<td>("+curr_data.lineinfo[0]+", "+curr_data.lineinfo[1]+")</td>";
+                                html_to_add += "<td><div class=\"row\"><button class=\"btn btn-xs btn-default\" data-toggle=\"popover\" data-placement=\"left\" data-content=\""+curr_data.content+"\">Preview content</button><a target=\"_blank\" href=\"{{ url_for('showsavedpaste') }}?paste="+curr_data.path+"&num=0\">  <button type=\"button\" class=\"btn btn-xs btn-info\">Show Paste</button></a></div></td>";
+
+                            html_to_add += "</tr>";
+                    }
+                    html_to_add += "</tbody>";
+                    html_to_add += "</table>";
+                    $("#mymodalbody").html(html_to_add);
+                    $("[data-toggle=popover]").popover();
+                    $("#button_show_plot").attr("href", "{{ url_for('terms_plot_tool')}}"+"?term="+the_modal.attr('data-term') );
+                } else {
+                    $("#mymodalbody").html("No paste containing this term has been received yet.");
+                    $("#button_show_plot").attr("href", "{{ url_for('terms_plot_tool')}}"+"?term="+the_modal.attr('data-term') );
+                }
+            });
+
+       });
+
+    $("#mymodal").on('hidden.bs.modal', function () {
+        $("#mymodalbody").html("<p>Loading paste information...</p>");
+        var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_for('static', filename='image/loading.gif') }}\" height='26' width='26' style='margin: 4px;'>";
+        $("#mymodalbody").append(loading_gif); // Show the loading GIF
+    });
+
+
+   });
+</script>
+
+<script>
+
+
+
+function perform_binding() {
+    $(".btn-interaction").unbind("click.interaction");
+    $(".btn-interaction").bind("click.interaction", perform_operation);
+}
+
+function perform_operation(){
+    var curr_section = $(this).attr('data-section');
+    var curr_action = $(this).attr('data-action'); 
+    if (curr_action == "add") {
+        var curr_term = $('#'+curr_section+'Input').val().toLowerCase();;
+    } else {
+        var curr_term = $(this).attr('data-content').toLowerCase();;
+    }
+    var data_to_send = { section: curr_section, action:curr_action, term: curr_term};
+
+    if (curr_term != "") {
+        console.log(data_to_send);
+        $.get("{{ url_for('terms_management_action') }}", data_to_send, function(data, status){
+            if(status == "success") {
+                var json = data;
+
+                if(json.section == "followTerm") {
+                    if(json.action == "add") {
+                        // query data
+                        $.get("{{ url_for('terms_management_query') }}", { term: json.term, section: json.section }, function(data2, status){
+                            var action_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"left\" title=\"Remove this term\" data-content=\"" + json.term + "\" data-section=\"followTerm\" data-action=\"delete\"><span class=\"glyphicon glyphicon-trash\"></span></button>"
+                            table_track.row.add( [ json.term, data2[3], data2[0], data2[1], data2[2], 0, action_button ] ).draw( false );
+                            perform_binding();
+                        });
+                   } else if (json.action == "delete") {
+                       // Find indexes of row which have the term in the first column
+                       var index = table_track.rows().eq( 0 ).filter( function (rowIdx) {
+                           return table_track.cell( rowIdx, 0 ).data() === json.term;
+                       } );
+                       table_track.rows(index).remove().draw( false );
+                    }
+                } else if(json.section == "blacklistTerm"){
+                    if(json.action == "add") {
+                        $.get("{{ url_for('terms_management_query') }}", { term: json.term, section: json.section }, function(data2, status){
+                            console.log(data2);
+                            var action_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Remove this term\" data-content=\"" + json.term + "\" data-section=\"blacklistTerm\" data-action=\"delete\"><span class=\"glyphicon glyphicon-trash\"></span></button>"
+                            table_black.row.add( [ json.term, data2[3], action_button ] ).draw( false );
+                            perform_binding();
+                        });
+                    } else if (json.action == "delete") {
+                       // Find indexes of row which have the term in the first column
+                       var index = table_black.rows().eq( 0 ).filter( function (rowIdx) {
+                           return table_black.cell( rowIdx, 0 ).data() === json.term;
+                       } );
+                       table_black.rows(index).remove().draw( false );
+                    }
+                }
+
+ 
+
+            }
+        });
+    }
+}
+</script>
+
+
+

var/www/templates/terms_plot_tool.html

@@ -0,0 +1,257 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+  <title>Analysis Information Leak framework Dashboard</title>
+
+  <!-- Core CSS -->
+  <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
+  <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
+  <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
+  <link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
+  <link href="{{ url_for('static', filename='css/jquery-ui.min.css') }}" rel="stylesheet" type="text/css" />
+  <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+  <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/jquery-ui.min.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
+
+  <style>
+      .sparkLineStats ul {
+          padding-left:0;
+          list-style:none
+      }
+  </style>
+</head>
+<body>
+
+   <div id="wrapper">
+      <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
+      {% include 'header.html' %}
+      <!-- /.navbar-top-links -->
+          <div class="navbar-default sidebar" role="navigation">
+              <div class="sidebar-collapse">
+                  <ul class="nav" id="side-menu">
+                      <li class="sidebar-search">
+                          {% include 'searchbox.html' %}
+                      </li>
+                 </ul>
+                  <!-- /#side-menu -->
+              </div>
+              <!-- /.sidebar-collapse -->
+           <a  href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
+          </div>
+          <!-- /.navbar-static-side -->
+      </nav>
+   </div>
+
+<div id="page-wrapper">
+  <div class="row">
+      <div class="col-lg-12">
+          <h1 class="page-header" data-page="page-termsfrequency" >Terms plot tool</h1>
+      </div>
+      <!-- /.col-lg-12 -->
+  </div>
+    <!-- /.row -->
+    <div class="row">
+
+    <!-- Panel OPTIONS -->
+    <div class="row">
+        <div class="col-lg-12">
+            <div class="row">
+                <div class="col-lg-12">
+                    <div id="panel-today" class="panel panel-default">
+                        <div class="panel-heading">
+                            <strong>Select options</strong>
+                        </div>
+                        <div class="panel-body">
+                            <div aria-disabled="false" class="slider sliderRange sliderBlue ui-slider ui-slider-horizontal ui-widget ui-widget-content ui-corner-all" style="margin-bottom: 5px;"></div>
+                            <strong>Date:</strong> <input type="text" id="amount" readonly style="border:0; color:#f6931f; font-weight:bold;">
+
+                            <div class="form-group input-group" style="margin-top: 30px;">
+                                <span class="input-group-addon"><span class="glyphicon glyphicon-stats"></span></span>
+                                <input id="TermInput" class="form-control" placeholder="Term to plot" type="text" style="max-width: 400px;" data-init-plot="{{ term }}">
+                                <button id="plot-btn" class="btn btn-info" style="margin-left: 10px;"><span class="fa fa-caret-down"> Plot a term</button>
+                                <button id="plot-btn-add" class="btn btn-success" style="margin-left: 6px;"><span class="fa fa-plus"></span> Add the term to the chart</button>
+                            </div>
+
+                        </div>
+                        <!-- /.panel-body -->
+                    </div>
+                </div>
+                <!-- /.panel -->
+            </div>
+            <!-- /.panel -->
+        </div>
+    </div>
+    
+    <!-- Panel PLOT -->
+    <div class="row">
+        <div class="col-lg-12">
+            <div class="row">
+                <div class="col-lg-12">
+                    <div id="panel-today" class="panel panel-default">
+                        <div class="panel-heading">
+                            <strong>Graph</strong>
+                        </div>
+                        <div class="panel-body">
+                            <div id="graph" style="height: 300px;"></div>
+                        </div>
+                        <!-- /.panel-body -->
+                    </div>
+                </div>
+                <!-- /.panel -->
+            </div>
+            <!-- /.panel -->
+        </div>
+    </div>
+
+
+    <!-- /.row -->
+    </div>
+<!-- /#page-wrapper -->
+</div>
+
+
+<div style="position: absolute; border: 1px solid rgb(255, 221, 221); padding: 2px; background-color: #333; color:white; opacity: 0.8; top: 423px; left: 616px; display: none;" id="tooltip"></div>
+
+
+<!-- import graph function -->
+<script>
+    $(document).ready(function(){
+        activePage = $('h1.page-header').attr('data-page');
+        $("#"+activePage).addClass("active");
+
+
+        /* Create the slider and button*/
+        $('#plot-btn-add').hide();
+        var today = Date.now();
+        var old_day = today - (12*31*24*60*60)*1000;
+        $( ".sliderRange" ).slider({
+            range: true,
+            min: old_day,
+            max: today,
+            values: [ today - (7*24*60*60)*1000, today ],
+            step: 24*60*60*1000,
+            slide: function( event, ui ) {
+                $( "#amount" ).val( new Date(ui.values[ 0 ]).toLocaleDateString() + " - " + new Date(ui.values[ 1 ]).toLocaleDateString() );
+            }
+        });
+
+        $( "#amount" ).val( new Date($( ".sliderRange" ).slider( "values", 0 )).toLocaleDateString() +
+          " - " + new Date($( ".sliderRange" ).slider( "values", 1 )).toLocaleDateString() );
+        
+        $('#plot-btn').click(plotData); 
+        $('#plot-btn-add').click(addData); 
+
+        $("#TermInput").val($("#TermInput").attr("data-init-plot"));
+        if($("#TermInput").attr("data-init-plot") != "") {
+            $("#plot-btn").click();
+        }
+
+        $("#TermInput").keyup(function(event){
+            if(event.keyCode == 13){
+                $("#plot-btn").click();
+                $("#TermInput").val("");
+            }
+        });
+
+   });
+</script>
+
+<script>
+var plot;
+var graph_data = [];
+var graph_options = {
+                        series: {
+                            lines: { 
+                                show: true,
+                                lineWidth: 2
+                            },
+                            bars: {show: false, barWidth: 60*60*1000},
+                            shadowSize: 0
+                        },
+                        grid: { 
+                            hoverable: true, 
+                            clickable: true, 
+                            tickColor: "#f9f9f9",
+                            borderWidth: 0
+                        },
+                        xaxis: {
+                              mode: "time",
+                              timeformat: "%m/%d",
+                              minTickSize: [1, "day"]
+                        },
+                        yaxis: {
+                            autoscaleMargin: 0.1,
+                        },
+                    }
+
+
+function plotData() {
+    $('#plot-btn-add').show("fast");
+    var curthis = $(this);
+    var term = $('#TermInput').val();
+    var range_start = new Date($( ".sliderRange" ).slider( "values", 0 )).getTime() / 1000;
+    var range_end = new Date($( ".sliderRange" ).slider( "values", 1 )).getTime() / 1000;
+
+    $.getJSON("{{ url_for('terms_plot_tool_data') }}", { range_start: range_start, range_end: range_end, term: term }, function(data, status){
+        graph_data = [];
+        var to_plot = [];
+        var curr_data = [];
+        for(i=0; i<data.length; i++) {
+                curr_data.push([data[i][0]*1000, data[i][1]]);
+        }
+        to_plot.push({ data: curr_data, label: term}); 
+        graph_data.push({ data: curr_data, label: term});
+        plot = $.plot($("#graph"), to_plot, graph_options);
+
+        $("#graph").bind("plothover", function (event, pos, item) {
+            if (item) {
+                var date = new Date(item.datapoint[0]);
+                var x = parseInt(date.getUTCMonth())+1 + "/" + date.getUTCDate();
+                var y = item.datapoint[1];
+    
+                $("#tooltip").html(item.series.label + " for "+x + " = " + y)
+                    .css({top: item.pageY-15, left: item.pageX+5})
+                    .fadeIn(200);
+            } else {
+                $("#tooltip").hide();
+            }
+        });
+
+        $("#TermInput").val("");
+    })
+}
+
+
+function addData() {
+    var curthis = $(this);
+    var term = $('#TermInput').val();
+    var range_start = new Date($( ".sliderRange" ).slider( "values", 0 )).getTime() / 1000;
+    var range_end = new Date($( ".sliderRange" ).slider( "values", 1 )).getTime() / 1000;
+
+    $.getJSON("{{ url_for('terms_plot_tool_data') }}", { range_start: range_start, range_end: range_end, term: term }, function(data, status){
+        var to_plot = [];
+        var curr_data = [];
+        for(i=0; i<data.length; i++) {
+                curr_data.push([data[i][0]*1000, data[i][1]]);
+        }
+        to_plot.push({ data: curr_data, label: term}); 
+        graph_data.push({ data: curr_data, label: term});
+        plot = $.plot($("#graph"), graph_data, graph_options);
+        $("#TermInput").val("");
+    })
+}
+
+</script>
+
+
+</body>
+</html>

var/www/templates/terms_plot_top.html

@@ -0,0 +1,576 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+  <title>Analysis Information Leak framework Dashboard</title>
+
+  <!-- Core CSS -->
+  <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
+  <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
+  <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
+  <link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
+  <link href="{{ url_for('static', filename='css/jquery-ui.min.css') }}" rel="stylesheet" type="text/css" />
+  <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+  <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
+
+  <style>
+      .sparkLineStats ul {
+          padding-left:0;
+          list-style:none
+      }
+      .table > tbody > tr > td {
+          padding: 5px;
+      }
+  </style>
+</head>
+<body>
+
+   <div id="wrapper">
+      <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
+      {% include 'header.html' %}
+      <!-- /.navbar-top-links -->
+          <div class="navbar-default sidebar" role="navigation">
+              <div class="sidebar-collapse">
+                  <ul class="nav" id="side-menu">
+                      <li class="sidebar-search">
+                          {% include 'searchbox.html' %}
+                      </li>
+                 </ul>
+                  <!-- /#side-menu -->
+              </div>
+              <!-- /.sidebar-collapse -->
+           <a  href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
+          </div>
+          <!-- /.navbar-static-side -->
+      </nav>
+   </div>
+
+<div id="page-wrapper">
+  <div class="row">
+      <div class="col-lg-12">
+          <h1 class="page-header" data-page="page-termsfrequency" >Terms frequency: Top set information</h1>
+      </div>
+      <!-- /.col-lg-12 -->
+  </div>
+    <!-- /.row -->
+    <div class="row">
+
+    <!-- Panel OPTIONS -->
+    <div class="row">
+        <div class="col-lg-12">
+            <div class="row">
+                <div class="col-lg-12">
+                    <div id="panel-today" class="panel panel-info">
+                        <div class="panel-heading">
+                            <strong>Today</strong>
+                        </div>
+                        <div class="panel-body">
+                            <div id="panel-today" class="panel panel-default">
+                                <div class="panel-heading">
+                                    <strong>Today top word</strong>
+                                </div>
+                                <div class="panel-body">
+                                    <div class="col-lg-6">
+                                    <table class="table table-striped">
+                                        <thead>
+                                            <tr>
+                                                <th>Term</th>
+                                                <th>Value</th>
+                                                <th>Action</th>
+                                                <th>Show</th>
+                                                <th data-toggle="tooltip" data-placement="left" title="Position of the term in the week and month set">Position</th>
+                                            </tr>
+                                        </thead>
+                                        <tbody id="table-today">
+                                        </tbody>
+                                    </table>
+                                    </div>
+                                    <div class="col-lg-6">
+                                    <table class="table table-striped">
+                                        <thead>
+                                            <tr>
+                                                <th>Term</th>
+                                                <th>Value</th>
+                                                <th>Action</th>
+                                                <th>Show</th>
+                                                <th data-toggle="tooltip" data-placement="left" title="Position of the term in the week and month set">Position</th>
+                                            </tr>
+                                        </thead>
+                                        <tbody id="table-today2">
+                                        </tbody>
+                                    </table>
+                                    </div>
+                                </div>
+                                <!-- /.panel-body -->
+                            </div>
+                            <div id="panel-today" class="panel panel-default">
+                                <div class="panel-heading">
+                                    <strong>Graph</strong>
+                                </div>
+                                <div class="panel-body">
+                                    <div id="graph-today" style="height: 400px;"></div>
+                                </div>
+                                <!-- /.panel-body -->
+                            </div>
+
+                        </div>
+                        <!-- /.panel-body -->
+                    </div>
+                </div>
+                <!-- /.panel -->
+            </div>
+            <!-- /.panel -->
+        </div>
+    </div>
+
+    <!-- Panel OPTIONS -->
+    <div class="row">
+        <div class="col-lg-12">
+            <div class="row">
+                <div class="col-lg-12">
+                    <div id="panel-today" class="panel panel-info">
+                        <div class="panel-heading">
+                            <strong>Week</strong>
+                        </div>
+                        <div class="panel-body">
+                            <div id="panel-week" class="panel panel-default">
+                                <div class="panel-heading">
+                                    <strong>Week top word</strong>
+                                </div>
+                                <div class="panel-body">
+                                    <div class="col-lg-6">
+                                     <table class="table table-striped">
+                                        <thead>
+                                            <tr>
+                                                <th>Term</th>
+                                                <th>Value</th>
+                                                <th>Action</th>
+                                                <th>Show</th>
+                                                <th data-toggle="tooltip" data-placement="left" title="Position of the term in the day and month set">Position</th>
+                                            </tr>
+                                        </thead>
+                                        <tbody id="table-week">
+                                        </tbody>
+                                    </table>
+                                    </div>
+                                    <div class="col-lg-6">
+                                    <table class="table table-striped">
+                                        <thead>
+                                            <tr>
+                                                <th>Term</th>
+                                                <th>Value</th>
+                                                <th>Action</th>
+                                                <th>Show</th>
+                                                <th data-toggle="tooltip" data-placement="left" title="Position of the term in the day and month set">Position</th>
+                                            </tr>
+                                        </thead>
+                                        <tbody id="table-week2">
+                                        </tbody>
+                                    </table>
+                                    </div>
+                                </div>
+                                <!-- /.panel-body -->
+                            </div>
+                            <div id="panel-week" class="panel panel-default">
+                                <div class="panel-heading">
+                                    <strong>Graph</strong>
+                                </div>
+                                <div class="panel-body">
+                                    <div id="graph-week" style="height: 400px;"></div>
+                                </div>
+                                <!-- /.panel-body -->
+                            </div>
+
+                        </div>
+                        <!-- /.panel-body -->
+                    </div>
+                </div>
+                <!-- /.panel -->
+            </div>
+            <!-- /.panel -->
+        </div>
+    </div>
+
+    <!-- Panel OPTIONS -->
+    <div class="row">
+        <div class="col-lg-12">
+            <div class="row">
+                <div class="col-lg-12">
+                    <div id="panel-today" class="panel panel-info">
+                        <div class="panel-heading">
+                            <strong>Month</strong>
+                        </div>
+                        <div class="panel-body">
+                            <div id="panel-month" class="panel panel-default">
+                                <div class="panel-heading">
+                                    <strong>Month top word</strong>
+                                </div>
+                                <div class="panel-body">
+                                    <div class="col-lg-6">
+                                     <table class="table table-striped">
+                                        <thead>
+                                            <tr>
+                                                <th>Term</th>
+                                                <th>Value</th>
+                                                <th>Action</th>
+                                                <th>Show</th>
+                                                <th data-toggle="tooltip" data-placement="left" title="Position of the term in the day and week set">Position</th>
+                                            </tr>
+                                        </thead>
+                                        <tbody id="table-month">
+                                        </tbody>
+                                    </table>
+                                    </div>
+                                    <div class="col-lg-6">
+                                    <table class="table table-striped">
+                                        <thead>
+                                            <tr>
+                                                <th>Term</th>
+                                                <th>Value</th>
+                                                <th>Action</th>
+                                                <th>Show</th>
+                                                <th data-toggle="tooltip" data-placement="left" title="Position of the term in the day and week set">Position</th>
+                                            </tr>
+                                        </thead>
+                                        <tbody id="table-month2">
+                                        </tbody>
+                                    </table>
+                                    </div>
+
+                                </div>
+                                <!-- /.panel-body -->
+                            </div>
+                            <div id="panel-month" class="panel panel-default">
+                                <div class="panel-heading">
+                                    <strong>Graph</strong>
+                                </div>
+                                <div class="panel-body">
+                                    <div id="graph-month" style="height: 400px;"></div>
+                                </div>
+                                <!-- /.panel-body -->
+                            </div>
+
+                        </div>
+                        <!-- /.panel-body -->
+                    </div>
+                </div>
+                <!-- /.panel -->
+            </div>
+            <!-- /.panel -->
+        </div>
+    </div>
+
+
+
+    <!-- /.row -->
+    </div>
+<!-- /#page-wrapper -->
+</div>
+
+<div style="position: absolute; border: 1px solid rgb(255, 221, 221); padding: 2px; background-color: #333; color:white; opacity: 0.8; top: 423px; left: 616px; display: none;" id="tooltip"></div>
+
+<!-- import graph function -->
+<script>
+    $(document).ready(function(){
+        activePage = $('h1.page-header').attr('data-page');
+        $("#"+activePage).addClass("active");
+   });
+</script>
+
+
+<script>
+var graph_options = {
+                        series: {
+                            lines: {
+                                show: true,
+                                lineWidth: 2
+                            },
+                            bars: {show: false, barWidth: 60*60*1000},
+                            shadowSize: 0
+                        },
+                        grid: {
+                            hoverable: true,
+                            clickable: true,
+                            tickColor: "#f9f9f9",
+                            borderWidth: 0
+                        },
+                        legend: { show: true,
+                                  noColumns: 1,
+                                  position: "nw",
+                                  labelFormatter: function(label, series) {
+                                      return "<a href=\"#\" onClick=\"hide_or_show2("+series.idx+", "+series.graphNum+"); return false;\" >" + label + "</a>";
+                                  }
+                        },
+                        xaxis: {
+                              mode: "time",
+                              timeformat: "%m/%d",
+                              minTickSize: [1, "day"]
+                        },
+                        yaxis: {
+                            //transform: function (v) { return v < 1 ? v : Math.log(v); }
+                            autoscaleMargin: 0.1,
+                        },
+                        tooltip: true,
+                        tooltipOpts: {
+                            content: " %s (%x.1 is %y.4) ",
+                            shifts: {
+                                x: -60,
+                                y: 25
+                            }
+                        }
+                    }
+
+
+set_today = "TopTermFreq_set_day";
+set_week = "TopTermFreq_set_week";
+set_month = "TopTermFreq_set_month";
+default_num_curves = 8;
+
+var plot_today;
+var plot_week;
+var plot_month;
+
+var promises = []; // Used to know when everything has been received
+
+promises.push($.getJSON("{{ url_for('terms_plot_top_data') }}", { set: set_today, num_day: 5 }, function(data, status){
+    data.sort(function(a, b){return b[2]-a[2];});
+    // Sort data
+    var table_today = $("#table-today")
+    var table_today2 = $("#table-today2")
+    var to_plot = [];
+    var unchecked_label = [];
+    for(i=0; i<data.length; i++) {
+        var highlight = data[i][3].week == "<20"? " style=\"background-color: lightgreen;\" " : "";
+        var curr_data = [];
+        for(j=0; j<data[i][1].length; j++) {
+            curr_data.push([data[i][1][j][0]*1000, data[i][1][j][1]]);
+        }
+        if (i>=default_num_curves) {
+            unchecked_label.push(data[i][0]);
+        }
+        to_plot.push({ data: curr_data, label: data[i][0], idx: i});
+        if ( i < (data.length/2))
+            table_today.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("today", data[i][0], i<default_num_curves)+"</td><td>"+data[i][3].week+", "+data[i][3].month+"</td></tr>");
+        else
+            table_today2.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("today", data[i][0], i<default_num_curves)+"</td><td>"+data[i][3].week+", "+data[i][3].month+"</td></tr>");
+    }
+    graph_options.series.graphNum=1;
+    plot_today = $.plot($("#graph-today"), to_plot, graph_options);
+    hide_unchecked_curves(plot_today, unchecked_label);
+    $("#graph-today").bind("plothover", function (event, pos, item) {
+        if (item) {
+            var date = new Date(item.datapoint[0]);
+            var x = parseInt(date.getUTCMonth())+1 + "/" + date.getUTCDate();
+            var y = item.datapoint[1];
+
+            $("#tooltip").html(item.series.label + " for "+x + " = " + y)
+                .css({top: item.pageY+5, left: item.pageX+5})
+                .fadeIn(200);
+        } else {
+            $("#tooltip").hide();
+        }
+    });
+
+}));
+
+promises.push($.getJSON("{{ url_for('terms_plot_top_data') }}", { set: set_week, num_day: 7 }, function(data, status){
+    data.sort(function(a, b){return b[2]-a[2];});
+    // Sort data
+    var table = $("#table-week")
+    var table2 = $("#table-week2")
+    var to_plot = [];
+    var unchecked_label = [];
+    for(i=0; i<data.length; i++) {
+        var highlight = parseInt(data[i][3].day) > 20? " style=\"background-color: orange;\" " : "";
+        var curr_data = [];
+        for(j=0; j<data[i][1].length; j++) {
+            curr_data.push([data[i][1][j][0]*1000, data[i][1][j][1]]);
+        }
+        if (i>=default_num_curves) {
+            unchecked_label.push(data[i][0]);
+        }
+        to_plot.push({ data: curr_data, label: data[i][0], idx: i});
+        if ( i < (data.length/2))
+            table.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("week", data[i][0], i<default_num_curves)+"</td><td><strong>"+data[i][3].day+"</strong>, "+data[i][3].month+"</td></tr>");
+        else
+            table2.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("week", data[i][0], i<default_num_curves)+"</td><td><strong>"+data[i][3].day+"</strong>, "+data[i][3].month+"</td></tr>");
+    }
+    graph_options.series.graphNum=2;
+    plot_week = $.plot($("#graph-week"), to_plot, graph_options);
+    hide_unchecked_curves(plot_week, unchecked_label);
+    $("#graph-week").bind("plothover", function (event, pos, item) {
+        if (item) {
+            var date = new Date(item.datapoint[0]);
+            var x = parseInt(date.getUTCMonth())+1 + "/" + date.getUTCDate();
+            var y = item.datapoint[1];
+
+            $("#tooltip").html(item.series.label + " for "+x + " = " + y)
+                .css({top: item.pageY+5, left: item.pageX+5})
+                .fadeIn(200);
+        } else {
+            $("#tooltip").hide();
+        }
+    });
+}));
+
+promises.push($.getJSON("{{ url_for('terms_plot_top_data') }}", { set: set_month, num_day: 31 }, function(data, status){
+    data.sort(function(a, b){return b[2]-a[2];});
+    // Sort data
+    var table = $("#table-month")
+    var table2 = $("#table-month2")
+    var to_plot = [];
+    var unchecked_label = [];
+    for(i=0; i<data.length; i++) {
+        var highlight = parseInt(data[i][3].day) > 20? " style=\"background-color: orange;\" " : "";
+        var curr_data = [];
+        for(j=0; j<data[i][1].length; j++) {
+            curr_data.push([data[i][1][j][0]*1000, data[i][1][j][1]]);
+        }
+        if (i>=default_num_curves) {
+            unchecked_label.push(data[i][0]);
+        }
+        to_plot.push({ data: curr_data, label: data[i][0], idx: i});
+        if ( i < (data.length/2))
+            table.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("month", data[i][0], i<default_num_curves)+"</td><td><strong>"+data[i][3].day+"</strong>, "+data[i][3].week+"</td></tr>");
+        else
+            table2.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("month", data[i][0], i<default_num_curves)+"</td><td><strong>"+data[i][3].day+"</strong>, "+data[i][3].week+"</td></tr>");
+    }
+    graph_options.series.graphNum=3;
+    plot_month = $.plot($("#graph-month"), to_plot, graph_options);
+    hide_unchecked_curves(plot_month, unchecked_label);
+    $("#graph-month").bind("plothover", function (event, pos, item) {
+        if (item) {
+            var date = new Date(item.datapoint[0]);
+            var x = parseInt(date.getUTCMonth())+1 + "/" + date.getUTCDate();
+            var y = item.datapoint[1];
+
+            $("#tooltip").html(item.series.label + " for "+x + " = " + y)
+                .css({top: item.pageY+5, left: item.pageX+5})
+                .fadeIn(200);
+        } else {
+            $("#tooltip").hide();
+        }
+    });
+}));
+
+/* When everything has been received, start adding tooltip */
+$.when.apply($, promises).done( function (arg) {
+    $('[data-toggle="tooltip"]').tooltip();
+
+    $(".btn-interaction").unbind("click.interaction");
+    $(".btn-interaction").bind("click.interaction", perform_operation);
+    $(".check-interaction").unbind("click.interaction");
+    $(".check-interaction").bind("click.interaction", hide_or_show);
+});
+
+
+function addbuttons(term) {
+    return "<button class=\"fa fa-eye btn-interaction btn btn-success\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Add to tracked list\" style=\"margin-right: 5px;\""+
+        "data-section=\"followTerm\" data-term=\""+term+"\"></button>"+
+        "<button class=\"fa fa-eye-slash btn-interaction btn btn-danger\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Add to black-list\""+
+        "data-section=\"blacklistTerm\" data-term=\""+term+"\"></button>";
+}
+
+function addcheckbox(graph, term, checked) {
+    var checked_text = checked ? "checked" : "";
+    return "<input type=checkbox "+checked_text+" class=\"check-interaction\" data-term=\""+term+"\" data-graph=\""+graph+"\"></input>";
+}
+
+function perform_operation(){
+    var curr_section = $(this).attr('data-section');
+    var curr_term = $(this).attr('data-term');
+
+    var data_to_send = { section: curr_section, action:"add", term: curr_term};
+
+    $.get("{{ url_for('terms_management_action') }}", data_to_send, function(data, status){
+        if(status == "success") {
+            location.reload();
+        }
+    });
+}
+
+
+function hide_unchecked_curves(plot, unchecked_label) {
+    var graphData = plot.getData();
+    var index;
+    for(i=0; i<graphData.length; i++) {
+        if($.inArray( graphData[i].label, unchecked_label ) != -1){
+            graphData[i].lines.show = false;
+        }
+    }
+    plot.setData(graphData);
+    plot.draw();
+}
+
+
+function hide_or_show() {
+    var curr_term = $(this).attr('data-term');
+    var graph = $(this).attr('data-graph');
+    var checked = $(this).prop('checked')
+
+    if(graph == "today") {
+        var graphData = plot_today.getData();
+        var index;
+        for(i=0; i<graphData.length; i++){
+            if(graphData[i].label == curr_term){
+                index = i;
+                break;
+            }
+        }
+        graphData[index].lines.show = checked;
+        plot_today.setData(graphData);
+        plot_today.draw();
+    } else if (graph == "week") {
+        var graphData = plot_week.getData();
+        var index;
+        for(i=0; i<graphData.length; i++){
+            if(graphData[i].label == curr_term){
+                index = i;
+                break;
+            }
+        }
+        graphData[index].lines.show = checked;
+        plot_week.setData(graphData);
+        plot_week.draw();
+    } else if (graph == "month") {
+          var graphData = plot_month.getData();
+        var index;
+        for(i=0; i<graphData.length; i++){
+            if(graphData[i].label == curr_term){
+                index = i;
+                break;
+            }
+        }
+        graphData[index].lines.show = checked;
+        plot_month.setData(graphData);
+        plot_month.draw();
+    }
+
+    // graph, hide curve
+}
+
+function hide_or_show2(index, graphNum) {
+    if (graphNum == 1)
+        var plot = plot_today;
+    else if (graphNum == 2)
+        var plot = plot_week;
+    else if (graphNum == 3)
+        var plot = plot_month;
+
+    var graphData = plot.getData();
+    graphData[index].lines.show = !graphData[index].lines.show;
+    plot.setData(graphData);
+    plot.draw();
+}
+</script>
+
+</body>
+</html>

var/www/update_thirdparty.sh

@@ -37,6 +37,15 @@ wget https://cdn.datatables.net/plug-ins/1.10.7/integration/bootstrap/3/dataTabl
 wget https://raw.githubusercontent.com/flot/flot/master/jquery.flot.js -O ./static/js/jquery.flot.js
 wget https://raw.githubusercontent.com/flot/flot/master/jquery.flot.pie.js -O ./static/js/jquery.flot.pie.js
 wget https://raw.githubusercontent.com/flot/flot/master/jquery.flot.time.js -O ./static/js/jquery.flot.time.js
+wget https://raw.githubusercontent.com/flot/flot/master/jquery.flot.stack.js -O ./static/js/jquery.flot.stack.js
+
+#Ressources for sparkline and canvasJS
+wget http://omnipotent.net/jquery.sparkline/2.1.2/jquery.sparkline.min.js -O ./static/js/jquery.sparkline.min.js
+wget http://canvasjs.com/fdm/chart/ -O temp/canvasjs.zip
+unzip temp/canvasjs.zip -d temp/
+mkdir temp
+mv temp/jquery.canvasjs.min.js ./static/js/jquery.canvasjs.min.js
+rm -rf temp
 
 mkdir -p ./static/image
 pushd static/image