Merge remote-tracking branch 'upstream/master'

.dockerignore

@@ -0,0 +1,2 @@
+*.git
+*.md

.travis.yml

@@ -5,27 +5,7 @@ python:
 
 sudo: required
 
-dist: trusty
+cache: pip
-
-addons:
-    apt:
-        packages:
-            # General dependencies
-            - python-pip
-            - python-virtualenv
-            - python-dev
-            - g++
-            - python-tk
-            - unzip
-            - libsnappy-dev
-            # Needed for bloomfilters
-            - libssl-dev
-            - python-numpy
-            - libfreetype6-dev
-            # Leveldb
-            - libgmp-dev
-            - libev-dev
-            - cmake
 
 env:
     - AIL_HOME=$TRAVIS_BUILD_DIR AIL_BIN=$TRAVIS_BUILD_DIR/bin/ \
@@ -35,49 +15,7 @@ env:
 
 
 install:
-    - pip install -U pip
+    - ./installing_deps.sh
-    # DNS
-    - sudo apt-get install -y libadns1 libadns1-dev screen
-    # required for mathplotlib
-    - test ! -L /usr/include/ft2build.h && sudo ln -s freetype2/ft2build.h /usr/include/
-    - pip install distribute
-    # Redis
-    - test ! -d redis/ && git clone https://github.com/antirez/redis.git
-    - pushd redis
-    - git checkout 3.2
-    - make
-    - popd
-    # Redis leveldb
-    - test ! -d redis-leveldb/ && git clone https://github.com/KDr2/redis-leveldb.git
-    - pushd redis-leveldb/
-    - git submodule init
-    - git submodule update
-    - make
-    - popd
-    # Faup
-    - test ! -d faup && git clone https://github.com/stricaud/faup.git
-    - pushd faup/
-    - test ! -d build && mkdir build
-    - cd build
-    - cmake .. && make
-    - sudo make install
-    - echo '/usr/local/lib' | sudo tee -a /etc/ld.so.conf.d/faup.conf
-    - sudo ldconfig
-    - popd
-    # PyFaup
-    - pushd faup/src/lib/bindings/python/
-    - python setup.py install
-    - popd
-    # Set config
-    - cp bin/packages/config.cfg.sample bin/packages/config.cfg
-    - mkdir -p $AIL_HOME/{PASTES,Blooms,dumps}
-    - mkdir -p $AIL_HOME/LEVEL_DB_DATA/{2016,2015,2014,2013}
-    - pip install -r pip_packages_requirement.txt
-    - python -m textblob.download_corpora
-    - pushd var/www/
-    - ./update_thirdparty.sh
-    - popd
-
 
 script:
     - pushd bin

Dockerfile

@@ -0,0 +1,9 @@
+FROM ubuntu:14.04
+
+RUN mkdir /opt/AIL && apt-get update -y \
+	&& apt-get install git python-dev build-essential \
+	libffi-dev libssl-dev libfuzzy-dev wget -y
+ADD . /opt/AIL
+WORKDIR /opt/AIL
+RUN ./installing_deps.sh && cd var/www/ && ./update_thirdparty.sh
+CMD bash docker_start.sh

README.md

@@ -7,7 +7,7 @@ AIL
 
 AIL framework - Framework for Analysis of Information Leaks
 
-AIL is a modular framework to analyse potential information leaks from unstructured data sources like pastes from Pastebin or similar services. AIL framework is flexible and can be extended to support other functionalities to mine sensitive information.
+AIL is a modular framework to analyse potential information leaks from unstructured data sources like pastes from Pastebin or similar services or unstructured data streams. AIL framework is flexible and can be extended to support other functionalities to mine sensitive information.
 
 ![Dashboard](./doc/screenshots/dashboard.png?raw=true "AIL framework dashboard")
 
@@ -38,7 +38,7 @@ Terms manager and occurence
 ![Term-Plot](./doc/screenshots/terms-plot.png?raw=true "AIL framework termPlot")
 
 
-AIL framework screencast: https://www.youtube.com/watch?v=9idfHCIMzBY
+[AIL framework screencast](https://www.youtube.com/watch?v=1_ZrZkRKmNo)
 
 Features
 --------
@@ -54,7 +54,7 @@ Features
 * A full-text indexer module to index unstructured information
 * Modules and web statistics 
 * Global sentiment analysis for each providers based on nltk vader module
-* Terms tracking and occurence
+* Terms tracking and occurrence
 * Many more modules for extracting phone numbers, credentials and others
 
 Installation
@@ -101,6 +101,37 @@ Eventually you can browse the status of the AIL framework website at the followi
 
         ``http://localhost:7000/``
 
+How to
+======
+
+How to feed the AIL framework
+-----------------------------
+
+For the moment, there are two different ways to feed AIL with data:
+
+1. Be a collaborator of CIRCL and ask to access our feed. It will be sent to the static IP your are using for AIL.
+
+2. You can setup [pystemon](https://github.com/CIRCL/pystemon) and use the custom feeder provided by AIL (see below).
+
+###Feeding AIL with pystemon
+AIL is an analysis tool, not a collector!
+However, if you want to collect some pastes and feed them to AIL, the procedure is described below.
+
+Nevertheless, moderate your queries!
+
+Here are the steps to setup pystemon and feed data to AIL:
+
+1. Clone the [pystemon's git repository](https://github.com/CIRCL/pystemon)
+
+2. Install its python dependencies inside your virtual environment
+
+3. Launch pystemon ``` ./pystemon ```
+
+4. Edit the file ```bin/feeder/pystemon-feeder.py``` and modify the pystemonpath path accordingly
+
+5. Launch pystemon-feeder ``` ./pystemon-feeder.py ```
+
+
 How to create a new module
 --------------------------
 
@@ -117,6 +148,10 @@ Feel free to fork the code, play with it, make some patches or add additional an
 
 To contribute your module, feel free to pull your contribution.
 
+Overview and License
+====================
+
+
 Redis and LevelDB overview
 --------------------------
 

bin/BrowseWarningPaste.py

@@ -24,7 +24,7 @@ if __name__ == "__main__":
     publisher.port = 6380
     publisher.channel = "Script"
 
-    config_section = 'Browse_warning_paste'
+    config_section = 'BrowseWarningPaste'
 
     p = Process(config_section)
 

bin/Credential.py

@@ -48,7 +48,7 @@ if __name__ == "__main__":
         if sites_set:
             message += ' Related websites: {}'.format(', '.join(sites_set))
 
-        to_print = 'Credential;{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, message)
+        to_print = 'Credential;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, message, paste.p_path)
 
         print('\n '.join(creds))
 

bin/CreditCards.py

@@ -63,14 +63,14 @@ if __name__ == "__main__":
                 to_print = 'CreditCard;{};{};{};'.format(
                     paste.p_source, paste.p_date, paste.p_name)
                 if (len(creditcard_set) > 0):
-                    publisher.warning('{}Checked {} valid number(s)'.format(
+                    publisher.warning('{}Checked {} valid number(s);{}'.format(
-                        to_print, len(creditcard_set)))
+                        to_print, len(creditcard_set), paste.p_path))
                     #Send to duplicate
                     p.populate_set_out(filename, 'Duplicate')
                     #send to Browse_warning_paste
                     p.populate_set_out('creditcard;{}'.format(filename), 'BrowseWarningPaste')
                 else:
-                    publisher.info('{}CreditCard related'.format(to_print))
+                    publisher.info('{}CreditCard related;{}'.format(to_print, paste.p_path))
         else:
             publisher.debug("Script creditcard is idling 1m")
             time.sleep(10)

bin/CurveManageTopSets.py

@@ -17,6 +17,7 @@ Requirements
 
 import redis
 import time
+import datetime
 import copy
 from pubsublogger import publisher
 from packages import lib_words
@@ -44,13 +45,14 @@ def manage_top_set():
     startDate = datetime.datetime.now()
     startDate = startDate.replace(hour=0, minute=0, second=0, microsecond=0)
     startDate = calendar.timegm(startDate.timetuple())
+    blacklist_size = int(server_term.scard(BlackListTermsSet_Name))
 
     dico = {}
 
-    # Retreive top data (2*max_card) from days sets
+    # Retreive top data (max_card + blacklist_size) from days sets
     for timestamp in range(startDate, startDate - top_termFreq_setName_month[1]*oneDay, -oneDay):
         curr_set = top_termFreq_setName_day[0] + str(timestamp)
-        array_top_day = server_term.zrevrangebyscore(curr_set, '+inf', '-inf', withscores=True, start=0, num=top_term_freq_max_set_cardinality*2)
+        array_top_day = server_term.zrevrangebyscore(curr_set, '+inf', '-inf', withscores=True, start=0, num=top_term_freq_max_set_cardinality+blacklist_size)
 
         for word, value in array_top_day:
             if word not in server_term.smembers(BlackListTermsSet_Name):
@@ -87,6 +89,11 @@ def manage_top_set():
     for elem in array_month:
         server_term.zadd(top_termFreq_setName_month[0], float(elem[1]), elem[0])
 
+    timestamp = int(time.mktime(datetime.datetime.now().timetuple()))
+    value = str(timestamp) + ", " + "-"
+    r_temp.set("MODULE_"+ "CurveManageTopSets" + "_" + str(os.getpid()), value)
+    print "refreshed module"
+
 
 
 if __name__ == '__main__':
@@ -105,6 +112,18 @@ if __name__ == '__main__':
     cfg = ConfigParser.ConfigParser()
     cfg.read(configfile)
 
+
+    # For Module Manager
+    r_temp = redis.StrictRedis(
+        host=cfg.get('RedisPubSub', 'host'),
+        port=cfg.getint('RedisPubSub', 'port'),
+        db=cfg.getint('RedisPubSub', 'db'))
+
+    timestamp = int(time.mktime(datetime.datetime.now().timetuple()))
+    value = str(timestamp) + ", " + "-"
+    r_temp.set("MODULE_"+ "CurveManageTopSets" + "_" + str(os.getpid()), value)
+    r_temp.sadd("MODULE_TYPE_"+ "CurveManageTopSets" , str(os.getpid()))
+
     server_term = redis.StrictRedis(
         host=cfg.get("Redis_Level_DB_TermFreq", "host"),
         port=cfg.getint("Redis_Level_DB_TermFreq", "port"),

bin/DomClassifier.py

@@ -51,13 +51,13 @@ def main():
                 localizeddomains = c.include(expression=cc_tld)
                 if localizeddomains:
                     print(localizeddomains)
-                    publisher.warning('DomainC;{};{};{};Checked {} located in {}'.format(
+                    publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format(
-                        PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc_tld))
+                        PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc_tld, PST.p_path))
                 localizeddomains = c.localizedomain(cc=cc)
                 if localizeddomains:
                     print(localizeddomains)
-                    publisher.warning('DomainC;{};{};{};Checked {} located in {}'.format(
+                    publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format(
-                        PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc))
+                        PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc, PST.p_path))
         except IOError:
             print "CRC Checksum Failed on :", PST.p_path
             publisher.error('Duplicate;{};{};{};CRC Checksum Failed'.format(

bin/Duplicates.py

@@ -131,8 +131,10 @@ if __name__ == "__main__":
                                 # index of paste
                                 index_current = r_serv_dico.get(dico_hash)
                                 paste_path = r_serv_dico.get(index_current)
+                                paste_date = r_serv_dico.get(index_current+'_date')
+                                paste_date = paste_date if paste_date != None else "No date available"
                                 if paste_path != None:
-                                    hash_dico[dico_hash] = (hash_type, paste_path, percent)
+                                    hash_dico[dico_hash] = (hash_type, paste_path, percent, paste_date)
 
                                 print '['+hash_type+'] '+'comparing: ' + str(PST.p_path[44:]) + '  and  ' + str(paste_path[44:]) + ' percentage: ' + str(percent)
                         except Exception,e:
@@ -142,6 +144,7 @@ if __name__ == "__main__":
             # Add paste in DB after checking to prevent its analysis twice
             # hash_type_i -> index_i  AND  index_i -> PST.PATH
             r_serv1.set(index, PST.p_path)
+            r_serv1.set(index+'_date', PST._get_p_date())
             r_serv1.sadd("INDEX", index)
             # Adding hashes in Redis
             for hash_type, paste_hash in paste_hashes.iteritems():
@@ -152,7 +155,7 @@ if __name__ == "__main__":
 
             # if there is data in this dictionnary
             if len(hash_dico) != 0:
-                # paste_tuple = (paste_path, percent)
+                # paste_tuple = (hash_type, date, paste_path, percent)
                 for dico_hash, paste_tuple in hash_dico.items():
                     dupl.append(paste_tuple)
 
@@ -162,7 +165,7 @@ if __name__ == "__main__":
                 if dupl != []:
                     PST.__setattr__("p_duplicate", dupl)
                     PST.save_attribute_redis("p_duplicate", dupl)
-                    publisher.info('{}Detected {}'.format(to_print, len(dupl)))
+                    publisher.info('{}Detected {};{}'.format(to_print, len(dupl), PST.p_path))
                     print '{}Detected {}'.format(to_print, len(dupl))
 
                 y = time.time()

bin/Global.py

@@ -59,7 +59,7 @@ if __name__ == '__main__':
             if int(time.time() - time_1) > 30:
                 to_print = 'Global; ; ; ;glob Processed {0} paste(s)'.format(processed_paste)
                 print to_print
-                publisher.info(to_print)
+                #publisher.info(to_print)
                 time_1 = time.time()
                 processed_paste = 0
             time.sleep(1)

bin/Helper.py

@@ -32,7 +32,7 @@ class PubSub(object):
         self.config.read(configfile)
         self.redis_sub = False
         self.zmq_sub = False
-        self.subscriber = None
+        self.subscribers = None
         self.publishers = {'Redis': [], 'ZMQ': []}
 
     def setup_subscribe(self, conn_name):
@@ -46,14 +46,19 @@ class PubSub(object):
                 host=self.config.get('RedisPubSub', 'host'),
                 port=self.config.get('RedisPubSub', 'port'),
                 db=self.config.get('RedisPubSub', 'db'))
-            self.subscriber = r.pubsub(ignore_subscribe_messages=True)
+            self.subscribers = r.pubsub(ignore_subscribe_messages=True)
-            self.subscriber.psubscribe(channel)
+            self.subscribers.psubscribe(channel)
         elif conn_name.startswith('ZMQ'):
             self.zmq_sub = True
             context = zmq.Context()
-            self.subscriber = context.socket(zmq.SUB)
+
-            self.subscriber.connect(self.config.get(conn_name, 'address'))
+            self.subscribers = []
-            self.subscriber.setsockopt(zmq.SUBSCRIBE, channel)
+            addresses = self.config.get(conn_name, 'address')
+            for address in addresses.split(','):
+                new_sub = context.socket(zmq.SUB)
+                new_sub.connect(address)
+                new_sub.setsockopt(zmq.SUBSCRIBE, channel)
+                self.subscribers.append(new_sub)
 
     def setup_publish(self, conn_name):
         if self.config.has_section(conn_name):
@@ -83,13 +88,18 @@ class PubSub(object):
 
     def subscribe(self):
         if self.redis_sub:
-            for msg in self.subscriber.listen():
+            for msg in self.subscribers.listen():
                 if msg.get('data', None) is not None:
                     yield msg['data']
         elif self.zmq_sub:
             while True:
-                msg = self.subscriber.recv()
+                for sub in self.subscribers:
-                yield msg.split(' ', 1)[1]
+                    try:
+                        msg = sub.recv(zmq.NOBLOCK)
+                        yield msg.split(' ', 1)[1]
+                    except zmq.error.Again as e:
+                        time.sleep(0.2)
+                        pass
         else:
             raise Exception('No subscribe function defined')
 
@@ -119,13 +129,7 @@ class Process(object):
             port=self.config.get('RedisPubSub', 'port'),
             db=self.config.get('RedisPubSub', 'db'))
 
-        self.moduleNum = 1
+        self.moduleNum = os.getpid()
-        for i in range(1, 50):
-            curr_num = self.r_temp.get("MODULE_"+self.subscriber_name + "_" + str(i))
-            if curr_num is None:
-                self.moduleNum = i
-                break
-
 
 
     def populate_set_in(self):
@@ -158,12 +162,14 @@ class Process(object):
                     path = "?"
                 value = str(timestamp) + ", " + path
                 self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum), value)
+                self.r_temp.sadd("MODULE_TYPE_"+self.subscriber_name, str(self.moduleNum))
                 return message
 
             except:
                 path = "?"
                 value = str(timestamp) + ", " + path
                 self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum), value)
+                self.r_temp.sadd("MODULE_TYPE_"+self.subscriber_name, str(self.moduleNum))
                 return message
 
     def populate_set_out(self, msg, channel=None):

bin/LAUNCH.sh

@@ -71,7 +71,9 @@ function launching_lvldb {
     db1_y='2013'
     db2_y='2014'
     db3_y='2016'
-    db4_y='3016'
+    db4_y='2017'
+
+    dbC_y='3016'
     nb_db=13
 
     screen -dmS "LevelDB"
@@ -83,10 +85,13 @@ function launching_lvldb {
     screen -S "LevelDB" -X screen -t "2014" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2014/ -P '$db2_y' -M '$nb_db'; read x'
     sleep 0.1
     screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db3_y' -M '$nb_db'; read x'
+    sleep 0.1
+    screen -S "LevelDB" -X screen -t "2017" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2017/ -P '$db4_y' -M '$nb_db'; read x'
+
 
     # For Curve
     sleep 0.1
-    screen -S "LevelDB" -X screen -t "3016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'3016/ -P '$db4_y' -M '$nb_db'; read x'
+    screen -S "LevelDB" -X screen -t "3016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'3016/ -P '$dbC_y' -M '$nb_db'; read x'
 }
 
 function launching_logs {
@@ -114,6 +119,8 @@ function launching_scripts {
 
     screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModuleInformation.py -k 0 -c 1; read x'
     sleep 0.1
+    screen -S "Script" -X screen -t "Mixer" bash -c './Mixer.py; read x'
+    sleep 0.1
     screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x'
     sleep 0.1
     screen -S "Script" -X screen -t "Duplicates" bash -c './Duplicates.py; read x'
@@ -158,7 +165,7 @@ function launching_scripts {
     sleep 0.1
     screen -S "Script" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x'
     sleep 0.1
-    screen -S "Script" -X screen -t "Browse_warning_paste" bash -c './Browse_warning_paste.py; read x'
+    screen -S "Script" -X screen -t "BrowseWarningPaste" bash -c './BrowseWarningPaste.py; read x'
     sleep 0.1
     screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x'
 

bin/Mail.py

@@ -55,9 +55,9 @@ if __name__ == "__main__":
                                              list(MX_values[1])))
 
                     pprint.pprint(MX_values)
-                    to_print = 'Mails;{};{};{};Checked {} e-mail(s)'.\
+                    to_print = 'Mails;{};{};{};Checked {} e-mail(s);{}'.\
                         format(PST.p_source, PST.p_date, PST.p_name,
-                               MX_values[0])
+                               MX_values[0], PST.p_path)
                     if MX_values[0] > is_critical:
                         publisher.warning(to_print)
                         #Send to duplicate

bin/Mixer.py

@@ -0,0 +1,195 @@
+#!/usr/bin/env python
+# -*-coding:UTF-8 -*
+"""
+The ZMQ_Feed_Q Module
+=====================
+
+This module is consuming the Redis-list created by the ZMQ_Feed_Q Module.
+
+This module take all the feeds provided in the config.
+Depending on the configuration, this module will process the feed as follow:
+    operation_mode 1: "Avoid any duplicate from any sources"
+        - The module maintain a list of content for each paste
+            - If the content is new, process it
+            - Else, do not process it but keep track for statistics on duplicate
+
+    operation_mode 2: "Keep duplicate coming from different sources"
+        - The module maintain a list of name given to the paste by the feeder
+            - If the name has not yet been seen, process it
+            - Elseif, the saved content associated with the paste is not the same, process it
+            - Else, do not process it but keep track for statistics on duplicate
+
+Note that the hash of the content is defined as the sha1(gzip64encoded).
+
+Every data coming from a named feed can be sent to a pre-processing module before going to the global module.
+The mapping can be done via the variable feed_queue_mapping
+
+Requirements
+------------
+
+*Need running Redis instances.
+*Need the ZMQ_Feed_Q Module running to be able to work properly.
+
+"""
+import base64
+import hashlib
+import os
+import time
+from pubsublogger import publisher
+import redis
+import ConfigParser
+
+from Helper import Process
+
+
+# CONFIG #
+refresh_time = 30
+feed_queue_mapping = { "feeder2": "preProcess1" } # Map a feeder name to a pre-processing module
+
+if __name__ == '__main__':
+    publisher.port = 6380
+    publisher.channel = 'Script'
+
+    config_section = 'Mixer'
+
+    p = Process(config_section)
+
+    configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
+    if not os.path.exists(configfile):
+        raise Exception('Unable to find the configuration file. \
+                        Did you set environment variables? \
+                        Or activate the virtualenv.')
+
+    cfg = ConfigParser.ConfigParser()
+    cfg.read(configfile)
+
+    # REDIS #
+    server = redis.StrictRedis(
+        host=cfg.get("Redis_Mixer_Cache", "host"),
+        port=cfg.getint("Redis_Mixer_Cache", "port"),
+        db=cfg.getint("Redis_Mixer_Cache", "db"))
+
+    # LOGGING #
+    publisher.info("Feed Script started to receive & publish.")
+
+    # OTHER CONFIG #
+    operation_mode = cfg.getint("Module_Mixer", "operation_mode")
+    ttl_key = cfg.getint("Module_Mixer", "ttl_duplicate")
+
+    # STATS #
+    processed_paste = 0
+    processed_paste_per_feeder = {}
+    duplicated_paste_per_feeder = {}
+    time_1 = time.time()
+
+
+    while True:
+
+        message = p.get_from_set()
+        if message is not None:
+            splitted = message.split()
+            if len(splitted) == 2:
+                complete_paste, gzip64encoded = splitted
+                try:
+                    feeder_name, paste_name = complete_paste.split('>')
+                    feeder_name.replace(" ","")
+                except ValueError as e:
+                    feeder_name = "unnamed_feeder"
+                    paste_name = complete_paste
+
+                # Processed paste
+                processed_paste += 1
+                try:
+                    processed_paste_per_feeder[feeder_name] += 1
+                except KeyError as e:
+                    # new feeder
+                    processed_paste_per_feeder[feeder_name] = 1
+                    duplicated_paste_per_feeder[feeder_name] = 0
+
+                relay_message = "{0} {1}".format(paste_name, gzip64encoded)
+                digest = hashlib.sha1(gzip64encoded).hexdigest()
+
+                # Avoid any duplicate coming from any sources
+                if operation_mode == 1:
+                    if server.exists(digest): # Content already exists
+                        #STATS
+                        duplicated_paste_per_feeder[feeder_name] += 1
+                    else: # New content
+
+                        # populate Global OR populate another set based on the feeder_name
+                        if feeder_name in feed_queue_mapping:
+                            p.populate_set_out(relay_message, feed_queue_mapping[feeder_name])
+                        else:
+                            p.populate_set_out(relay_message, 'Mixer')
+
+                    server.sadd(digest, feeder_name)
+                    server.expire(digest, ttl_key)
+
+
+                # Keep duplicate coming from different sources
+                else:
+                    # Filter to avoid duplicate 
+                    content = server.get('HASH_'+paste_name)
+                    if content is None:
+                        # New content
+                        # Store in redis for filtering
+                        server.set('HASH_'+paste_name, digest)
+                        server.sadd(paste_name, feeder_name)
+                        server.expire(paste_name, ttl_key)
+                        server.expire('HASH_'+paste_name, ttl_key)
+
+                        # populate Global OR populate another set based on the feeder_name
+                        if feeder_name in feed_queue_mapping:
+                            p.populate_set_out(relay_message, feed_queue_mapping[feeder_name])
+                        else:
+                            p.populate_set_out(relay_message, 'Mixer')
+
+                    else:
+                        if digest != content:
+                            # Same paste name but different content
+                            #STATS
+                            duplicated_paste_per_feeder[feeder_name] += 1
+                            server.sadd(paste_name, feeder_name)
+                            server.expire(paste_name, ttl_key)
+
+                            # populate Global OR populate another set based on the feeder_name
+                            if feeder_name in feed_queue_mapping:
+                                p.populate_set_out(relay_message, feed_queue_mapping[feeder_name])
+                            else:
+                                p.populate_set_out(relay_message, 'Mixer')
+
+                        else:
+                            # Already processed
+                            # Keep track of processed pastes
+                            #STATS
+                            duplicated_paste_per_feeder[feeder_name] += 1
+                            continue
+
+            else:
+                # TODO Store the name of the empty paste inside a Redis-list.
+                print "Empty Paste: not processed"
+                publisher.debug("Empty Paste: {0} not processed".format(message))
+        else:
+            print "Empty Queues: Waiting..."
+            if int(time.time() - time_1) > refresh_time:
+                print processed_paste_per_feeder
+                to_print = 'Mixer; ; ; ;mixer_all All_feeders Processed {0} paste(s) in {1}sec'.format(processed_paste, refresh_time)
+                print to_print
+                publisher.info(to_print)
+                processed_paste = 0
+
+                for feeder, count in processed_paste_per_feeder.iteritems():
+                    to_print = 'Mixer; ; ; ;mixer_{0} {0} Processed {1} paste(s) in {2}sec'.format(feeder, count, refresh_time)
+                    print to_print
+                    publisher.info(to_print)
+                    processed_paste_per_feeder[feeder] = 0
+
+                for feeder, count in duplicated_paste_per_feeder.iteritems():
+                    to_print = 'Mixer; ; ; ;mixer_{0} {0} Duplicated {1} paste(s) in {2}sec'.format(feeder, count, refresh_time)
+                    print to_print
+                    publisher.info(to_print)
+                    duplicated_paste_per_feeder[feeder] = 0
+
+                time_1 = time.time()
+            time.sleep(0.5)
+            continue

bin/ModuleInformation.py

@@ -24,60 +24,165 @@ import ConfigParser
 import json
 from terminaltables import AsciiTable
 import textwrap
+from colorama import Fore, Back, Style, init
+import curses
 
 # CONFIG VARIABLES
-threshold_stucked_module = 60*60*1 #1 hour
+kill_retry_threshold = 60 #1m
 log_filename = "../logs/moduleInfo.log"
 command_search_pid = "ps a -o pid,cmd | grep {}"
+command_search_name = "ps a -o pid,cmd | grep {}"
 command_restart_module = "screen -S \"Script\" -X screen -t \"{}\" bash -c \"./{}.py; read x\""
 
+init() #Necesary for colorama
+printarrayGlob = [None]*14
+printarrayGlob.insert(0, ["Time", "Module", "PID", "Action"])
+lastTimeKillCommand = {}
+
+#Curses init
+#stdscr = curses.initscr()
+#curses.cbreak()
+#stdscr.keypad(1)
+
+# GLOBAL
+last_refresh = 0
+
 
 def getPid(module):
     p = Popen([command_search_pid.format(module+".py")], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
     for line in p.stdout:
+        print line
         splittedLine = line.split()
         if 'python2' in splittedLine:
             return int(splittedLine[0])
-        else:
+    return None
-            return None
 
 def clearRedisModuleInfo():
     for k in server.keys("MODULE_*"):
         server.delete(k)
+    inst_time = datetime.datetime.fromtimestamp(int(time.time()))
+    printarrayGlob.insert(1, [inst_time, "*", "-", "Cleared redis module info"])
+    printarrayGlob.pop()
 
-def kill_module(module):
+def cleanRedis():
+    for k in server.keys("MODULE_TYPE_*"):
+        moduleName = k[12:].split('_')[0]
+        for pid in server.smembers(k):
+            flag_pid_valid = False
+            proc = Popen([command_search_name.format(pid)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
+            for line in proc.stdout:
+                splittedLine = line.split()
+                if ('python2' in splittedLine or 'python' in splittedLine) and "./"+moduleName+".py" in splittedLine:
+                    flag_pid_valid = True
+
+            if not flag_pid_valid:
+                print flag_pid_valid, 'cleaning', pid, 'in', k
+                server.srem(k, pid)
+                inst_time = datetime.datetime.fromtimestamp(int(time.time()))
+                printarrayGlob.insert(1, [inst_time, moduleName, pid, "Cleared invalid pid in " + k])
+                printarrayGlob.pop()
+                #time.sleep(5)
+
+
+def kill_module(module, pid):
     print ''
     print '-> trying to kill module:', module
 
-    pid = getPid(module)
+    if pid is None:
+        print 'pid was None'
+        printarrayGlob.insert(1, [0, module, pid, "PID was None"])
+        printarrayGlob.pop()
+        pid = getPid(module)
+    else: #Verify that the pid is at least in redis
+        if server.exists("MODULE_"+module+"_"+str(pid)) == 0:
+            return
+
+    lastTimeKillCommand[pid] = int(time.time())
     if pid is not None:
-        os.kill(pid, signal.SIGUSR1)
+        try:
+            os.kill(pid, signal.SIGUSR1)
+        except OSError:
+            print pid, 'already killed'
+            inst_time = datetime.datetime.fromtimestamp(int(time.time()))
+            printarrayGlob.insert(1, [inst_time, module, pid, "Already killed"])
+            printarrayGlob.pop()
+            return
         time.sleep(1)
         if getPid(module) is None:
             print module, 'has been killed'
             print 'restarting', module, '...'
             p2 = Popen([command_restart_module.format(module, module)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
+            inst_time = datetime.datetime.fromtimestamp(int(time.time()))
+            printarrayGlob.insert(1, [inst_time, module, pid, "Killed"])
+            printarrayGlob.insert(1, [inst_time, module, "?", "Restarted"])
+            printarrayGlob.pop()
+            printarrayGlob.pop()
 
         else:
             print 'killing failed, retrying...'
-            time.sleep(3)
+            inst_time = datetime.datetime.fromtimestamp(int(time.time()))
+            printarrayGlob.insert(1, [inst_time, module, pid, "Killing #1 failed."])
+            printarrayGlob.pop()
+
+            time.sleep(1)
             os.kill(pid, signal.SIGUSR1)
             time.sleep(1)
             if getPid(module) is None:
                 print module, 'has been killed'
                 print 'restarting', module, '...'
                 p2 = Popen([command_restart_module.format(module, module)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
+                inst_time = datetime.datetime.fromtimestamp(int(time.time()))
+                printarrayGlob.insert(1, [inst_time, module, pid, "Killed"])
+                printarrayGlob.insert(1, [inst_time, module, "?", "Restarted"])
+                printarrayGlob.pop()
+                printarrayGlob.pop()
             else:
                 print 'killing failed!'
-    time.sleep(7)
+                inst_time = datetime.datetime.fromtimestamp(int(time.time()))
+                printarrayGlob.insert(1, [inst_time, module, pid, "Killing failed!"])
+                printarrayGlob.pop()
+    else:
+        print 'Module does not exist'
+        inst_time = datetime.datetime.fromtimestamp(int(time.time()))
+        printarrayGlob.insert(1, [inst_time, module, pid, "Killing failed, module not found"])
+        printarrayGlob.pop()
+    #time.sleep(5)
+    cleanRedis()
+
+def get_color(time, idle):
+    if time is not None:
+        temp = time.split(':')
+        time = int(temp[0])*3600 + int(temp[1])*60 + int(temp[2])
+
+        if time >= args.treshold:
+            if not idle:
+                return Back.RED + Style.BRIGHT
+            else:
+                return Back.MAGENTA + Style.BRIGHT
+        elif time > args.treshold/2:
+            return Back.YELLOW + Style.BRIGHT
+        else:
+            return Back.GREEN + Style.BRIGHT
+    else:
+        return Style.RESET_ALL
+
+def waiting_refresh():
+    global last_refresh
+    if time.time() - last_refresh < args.refresh:
+        return False
+    else:
+        last_refresh = time.time()
+        return True
+
     
 
 if __name__ == "__main__":
 
     parser = argparse.ArgumentParser(description='Show info concerning running modules and log suspected stucked modules. May be use to automatically kill and restart stucked one.')
     parser.add_argument('-r', '--refresh', type=int, required=False, default=1, help='Refresh rate')
-    parser.add_argument('-k', '--autokill', type=int, required=True, default=1, help='Enable auto kill option (1 for TRUE, anything else for FALSE)')
+    parser.add_argument('-t', '--treshold', type=int, required=False, default=60*10*1, help='Refresh rate')
-    parser.add_argument('-c', '--clear', type=int, required=False, default=1, help='Clear the current module information (Used to clear data from old launched modules)')
+    parser.add_argument('-k', '--autokill', type=int, required=False, default=0, help='Enable auto kill option (1 for TRUE, anything else for FALSE)')
+    parser.add_argument('-c', '--clear', type=int, required=False, default=0, help='Clear the current module information (Used to clear data from old launched modules)')
 
     args = parser.parse_args()
 
@@ -99,100 +204,150 @@ if __name__ == "__main__":
     if args.clear == 1:
         clearRedisModuleInfo()
 
+    lastTime = datetime.datetime.now()
 
     module_file_array = set()
-    with open('../doc/all_modules.txt', 'r') as module_file:
+    no_info_modules = {}
+    path_allmod = os.path.join(os.environ['AIL_HOME'], 'doc/all_modules.txt')
+    with open(path_allmod, 'r') as module_file:
         for line in module_file:
             module_file_array.add(line[:-1])
 
+        cleanRedis()
+
         while True:
+            if waiting_refresh():
 
-            all_queue = set()
+                #key = ''
-            curr_range = 50
+                #while key != 'q':
-            printarray1 = []
+                #    key = stdsrc.getch()
-            printarray2 = []
+                #    stdscr.refresh()
-            printarray3 = []
-            for queue, card in server.hgetall("queues").iteritems():
-                all_queue.add(queue)
-                key = "MODULE_" + queue + "_"
-                for i in range(1, 50):
-                    curr_num = server.get("MODULE_"+ queue + "_" + str(i))
-                    if curr_num is None:
-                        curr_range = i
-                        break
     
-                for moduleNum in range(1, curr_range):
+                all_queue = set()
-                    value = server.get(key + str(moduleNum))
+                printarray1 = []
-                    if value is not None:
+                printarray2 = []
-                        timestamp, path = value.split(", ")
+                printarray3 = []
-                        if timestamp is not None and path is not None:
+                for queue, card in server.hgetall("queues").iteritems():
-                            startTime_readable = datetime.datetime.fromtimestamp(int(timestamp))
+                    all_queue.add(queue)
-                            processed_time_readable = str((datetime.datetime.now() - startTime_readable)).split('.')[0]
+                    key = "MODULE_" + queue + "_"
+                    keySet = "MODULE_TYPE_" + queue
+                    array_module_type = []
     
-                            if int(card) > 0:
+                    for moduleNum in server.smembers(keySet):
-                                if int((datetime.datetime.now() - startTime_readable).total_seconds()) > threshold_stucked_module:
+                        value = server.get(key + str(moduleNum))
-                                    log = open(log_filename, 'a')
+                        if value is not None:
-                                    log.write(json.dumps([queue, card, str(startTime_readable), str(processed_time_readable), path]) + "\n")
+                            timestamp, path = value.split(", ")
-                                    if args.autokill == 1:
+                            if timestamp is not None and path is not None:
-                                        kill_module(queue)
+                                startTime_readable = datetime.datetime.fromtimestamp(int(timestamp))
+                                processed_time_readable = str((datetime.datetime.now() - startTime_readable)).split('.')[0]
     
-                                printarray1.append([str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path)])
+                                if int(card) > 0:
+                                    if int((datetime.datetime.now() - startTime_readable).total_seconds()) > args.treshold:
+                                        log = open(log_filename, 'a')
+                                        log.write(json.dumps([queue, card, str(startTime_readable), str(processed_time_readable), path]) + "\n")
+                                        try:
+                                            last_kill_try = time.time() - lastTimeKillCommand[moduleNum]
+                                        except KeyError:
+                                            last_kill_try = kill_retry_threshold+1
+                                        if args.autokill == 1 and last_kill_try > kill_retry_threshold :
+                                            kill_module(queue, int(moduleNum))
     
+                                    array_module_type.append([get_color(processed_time_readable, False) + str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path) + get_color(None, False)])
+    
+                                else:
+                                    printarray2.append([get_color(processed_time_readable, True) + str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path) + get_color(None, True)])
+                            array_module_type.sort(lambda x,y: cmp(x[4], y[4]), reverse=True)
+                    for e in array_module_type:
+                        printarray1.append(e)
+    
+                for curr_queue in module_file_array:
+                    if curr_queue not in all_queue:
+                            printarray3.append([curr_queue, "Not running"])
+                    else:
+                        if len(list(server.smembers('MODULE_TYPE_'+curr_queue))) == 0:
+                            if curr_queue not in no_info_modules:
+                                no_info_modules[curr_queue] = int(time.time())
+                                printarray3.append([curr_queue, "No data"])
                             else:
-                                printarray2.append([str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path)])
+                                #If no info since long time, try to kill
+                                if args.autokill == 1:
+                                    if int(time.time()) - no_info_modules[curr_queue] > args.treshold:
+                                        kill_module(curr_queue, None)
+                                        no_info_modules[curr_queue] = int(time.time())
+                                    printarray3.append([curr_queue, "Stuck or idle, restarting in " + str(abs(args.treshold - (int(time.time()) - no_info_modules[curr_queue]))) + "s"])
+                                else:
+                                    printarray3.append([curr_queue, "Stuck or idle, restarting disabled"])
     
-            for curr_queue in module_file_array:
+                ## FIXME To add:
-                if curr_queue not in all_queue:
+                ## Button KILL Process using  Curses
-                    printarray3.append([curr_queue, "Not running"])
     
-            printarray1.sort(lambda x,y: cmp(x[4], y[4]), reverse=True)
+                printarray1.sort(key=lambda x: x[0][9:], reverse=False)
-            printarray2.sort(lambda x,y: cmp(x[4], y[4]), reverse=True)
+                printarray2.sort(key=lambda x: x[0][9:], reverse=False)
-            printarray1.insert(0,["Queue", "#", "Amount", "Paste start time", "Processing time for current paste (H:M:S)", "Paste hash"])
+                printarray1.insert(0,["Queue", "PID", "Amount", "Paste start time", "Processing time for current paste (H:M:S)", "Paste hash"])
-            printarray2.insert(0,["Queue", "#","Amount", "Paste start time", "Time since idle (H:M:S)", "Last paste hash"])
+                printarray2.insert(0,["Queue", "PID","Amount", "Paste start time", "Time since idle (H:M:S)", "Last paste hash"])
-            printarray3.insert(0,["Queue", "State"])
+                printarray3.insert(0,["Queue", "State"])
     
-            os.system('clear')
+                os.system('clear')
-            t1 = AsciiTable(printarray1, title="Working queues")
+                t1 = AsciiTable(printarray1, title="Working queues")
-            t1.column_max_width(1)
+                t1.column_max_width(1)
-            if not t1.ok:
+                if not t1.ok:
-                    longest_col = t1.column_widths.index(max(t1.column_widths))
+                        longest_col = t1.column_widths.index(max(t1.column_widths))
-                    max_length_col = t1.column_max_width(longest_col)
+                        max_length_col = t1.column_max_width(longest_col)
-                    if max_length_col > 0:
+                        if max_length_col > 0:
-                        for i, content in enumerate(t1.table_data):
+                            for i, content in enumerate(t1.table_data):
-                            if len(content[longest_col]) > max_length_col:
+                                if len(content[longest_col]) > max_length_col:
-                                temp = ''
+                                    temp = ''
-                                for l in content[longest_col].splitlines():
+                                    for l in content[longest_col].splitlines():
-                                    if len(l) > max_length_col:
+                                        if len(l) > max_length_col:
-                                        temp += '\n'.join(textwrap.wrap(l, max_length_col)) + '\n'
+                                            temp += '\n'.join(textwrap.wrap(l, max_length_col)) + '\n'
-                                    else:
+                                        else:
-                                        temp += l + '\n'
+                                            temp += l + '\n'
-                                    content[longest_col] = temp.strip()
+                                        content[longest_col] = temp.strip()
-                            t1.table_data[i] = content
+                                t1.table_data[i] = content
     
-            t2 = AsciiTable(printarray2, title="Idling queues")
+                t2 = AsciiTable(printarray2, title="Idling queues")
-            t2.column_max_width(1)
+                t2.column_max_width(1)
-            if not t2.ok:
+                if not t2.ok:
-                    longest_col = t2.column_widths.index(max(t2.column_widths))
+                        longest_col = t2.column_widths.index(max(t2.column_widths))
-                    max_length_col = t2.column_max_width(longest_col)
+                        max_length_col = t2.column_max_width(longest_col)
-                    if max_length_col > 0:
+                        if max_length_col > 0:
-                        for i, content in enumerate(t2.table_data):
+                            for i, content in enumerate(t2.table_data):
-                            if len(content[longest_col]) > max_length_col:
+                                if len(content[longest_col]) > max_length_col:
-                                temp = ''
+                                    temp = ''
-                                for l in content[longest_col].splitlines():
+                                    for l in content[longest_col].splitlines():
-                                    if len(l) > max_length_col:
+                                        if len(l) > max_length_col:
-                                        temp += '\n'.join(textwrap.wrap(l, max_length_col)) + '\n'
+                                            temp += '\n'.join(textwrap.wrap(l, max_length_col)) + '\n'
-                                    else:
+                                        else:
-                                        temp += l + '\n'
+                                            temp += l + '\n'
-                                    content[longest_col] = temp.strip()
+                                        content[longest_col] = temp.strip()
-                            t2.table_data[i] = content
+                                t2.table_data[i] = content
     
-            t3 = AsciiTable(printarray3, title="Not running queues")
+                t3 = AsciiTable(printarray3, title="Not running queues")
-            t3.column_max_width(1)
+                t3.column_max_width(1)
     
-            print t1.table
+                printarray4 = []
-            print '\n'
+                for elem in printarrayGlob:
-            print t2.table
+                    if elem is not None:
-            print '\n'
+                        printarray4.append(elem)
-            print t3.table
     
-            time.sleep(args.refresh)
+                t4 = AsciiTable(printarray4, title="Last actions")
+                t4.column_max_width(1)
+    
+                legend_array = [["Color", "Meaning"], [Back.RED+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time >=" +str(args.treshold)+Style.RESET_ALL], [Back.MAGENTA+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time >=" +str(args.treshold)+" while idle"+Style.RESET_ALL], [Back.YELLOW+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time >=" +str(args.treshold/2)+Style.RESET_ALL], [Back.GREEN+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time <" +str(args.treshold)]]
+                legend = AsciiTable(legend_array, title="Legend")
+                legend.column_max_width(1)
+    
+                print legend.table
+                print '\n'
+                print t1.table
+                print '\n'
+                print t2.table
+                print '\n'
+                print t3.table
+                print '\n'
+                print t4.table
+    
+                if (datetime.datetime.now() - lastTime).total_seconds() > args.refresh*5: 
+                    lastTime = datetime.datetime.now()
+                    cleanRedis()
+                #time.sleep(args.refresh)

bin/Onion.py

@@ -133,8 +133,8 @@ if __name__ == "__main__":
                                                     PST.p_name)
                 if len(domains_list) > 0:
 
-                    publisher.warning('{}Detected {} .onion(s)'.format(
+                    publisher.warning('{}Detected {} .onion(s);{}'.format(
-                        to_print, len(domains_list)))
+                        to_print, len(domains_list),PST.p_path))
                     now = datetime.datetime.now()
                     path = os.path.join('onions', str(now.year).zfill(4),
                                         str(now.month).zfill(2),
@@ -144,9 +144,9 @@ if __name__ == "__main__":
                                                         PST.p_date,
                                                         PST.p_name)
                     for url in fetch(p, r_cache, urls, domains_list, path):
-                        publisher.warning('{}Checked {}'.format(to_print, url))
+                        publisher.warning('{}Checked {};{}'.format(to_print, url, PST.p_path))
                 else:
-                    publisher.info('{}Onion related'.format(to_print))
+                    publisher.info('{}Onion related;{}'.format(to_print, PST.p_path))
 
             prec_filename = filename
         else:

bin/Release.py

@@ -34,7 +34,7 @@ if __name__ == "__main__":
         if len(releases) == 0:
             continue
 
-        to_print = 'Release;{};{};{};{} releases'.format(paste.p_source, paste.p_date, paste.p_name, len(releases))
+        to_print = 'Release;{};{};{};{} releases;{}'.format(paste.p_source, paste.p_date, paste.p_name, len(releases), paste.p_path)
         if len(releases) > 30:
             publisher.warning(to_print)
         else:

bin/SQLInjectionDetection.py

@@ -69,7 +69,7 @@ def analyse(url, path):
         if (result_path > 1) or (result_query > 1):
             print "Detected SQL in URL: "
             print urllib2.unquote(url)
-            to_print = 'SQLInjection;{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL")
+            to_print = 'SQLInjection;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL", paste.p_path)
             publisher.warning(to_print)
             #Send to duplicate
             p.populate_set_out(path, 'Duplicate')

bin/SentimentAnalysis.py

@@ -32,6 +32,20 @@ accepted_Mime_type = ['text/plain']
 size_threshold = 250
 line_max_length_threshold = 1000
 
+import os
+import ConfigParser
+
+configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
+if not os.path.exists(configfile):
+    raise Exception('Unable to find the configuration file. \
+        Did you set environment variables? \
+        Or activate the virtualenv.')
+
+cfg = ConfigParser.ConfigParser()
+cfg.read(configfile)
+
+sentiment_lexicon_file = cfg.get("Directories", "sentiment_lexicon_file")
+
 def Analyse(message, server):
     path = message
     paste = Paste.Paste(path)
@@ -61,7 +75,7 @@ def Analyse(message, server):
             avg_score = {'neg': 0.0, 'neu': 0.0, 'pos': 0.0, 'compoundPos': 0.0, 'compoundNeg': 0.0}
             neg_line = 0
             pos_line = 0
-            sid = SentimentIntensityAnalyzer()
+            sid = SentimentIntensityAnalyzer(sentiment_lexicon_file)
             for sentence in sentences:
                  ss = sid.polarity_scores(sentence)
                  for k in sorted(ss):
@@ -110,6 +124,16 @@ def isJSON(content):
     except Exception,e:
         return False
 
+import signal
+
+class TimeoutException(Exception):
+    pass
+
+def timeout_handler(signum, frame):
+    raise TimeoutException
+
+signal.signal(signal.SIGALRM, timeout_handler)
+
 if __name__ == '__main__':
     # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
     # Port of the redis instance used by pubsublogger
@@ -138,6 +162,12 @@ if __name__ == '__main__':
             publisher.debug("{} queue is empty, waiting".format(config_section))
             time.sleep(1)
             continue
-
+        signal.alarm(60)
-        Analyse(message, server)
+        try:
+            Analyse(message, server)
+        except TimeoutException:
+            print ("{0} processing timeout".format(message))
+            continue
+        else:
+            signal.alarm(0)
 

bin/Tokenize.py

@@ -28,6 +28,15 @@ from packages import Paste
 from pubsublogger import publisher
 
 from Helper import Process
+import signal
+
+class TimeoutException(Exception):
+    pass
+
+def timeout_handler(signum, frame):
+    raise TimeoutException
+
+signal.signal(signal.SIGALRM, timeout_handler)
 
 if __name__ == "__main__":
     publisher.port = 6380
@@ -44,10 +53,17 @@ if __name__ == "__main__":
         print message
         if message is not None:
             paste = Paste.Paste(message)
-            for word, score in paste._get_top_words().items():
+            signal.alarm(5)
-                if len(word) >= 4:
+            try:
-                    msg = '{} {} {}'.format(paste.p_path, word, score)
+                for word, score in paste._get_top_words().items():
-                    p.populate_set_out(msg)
+                    if len(word) >= 4:
+                        msg = '{} {} {}'.format(paste.p_path, word, score)
+                        p.populate_set_out(msg)
+            except TimeoutException:
+                 print ("{0} processing timeout".format(paste.p_path))
+                 continue
+            else:
+                signal.alarm(0)
         else:
             publisher.debug("Tokeniser is idling 10s")
             time.sleep(10)

bin/Web.py

@@ -113,7 +113,7 @@ if __name__ == "__main__":
                     # IP allocation)
                     if cc is not None and cc != "EU":
                         print hostl, asn, cc, \
-                            pycountry.countries.get(alpha2=cc).name
+                            pycountry.countries.get(alpha_2=cc).name
                         if cc == cc_critical:
                             to_print = 'Url;{};{};{};Detected {} {}'.format(
                                     PST.p_source, PST.p_date, PST.p_name,
@@ -131,8 +131,8 @@ if __name__ == "__main__":
                                              list(A_values[1])))
 
                     pprint.pprint(A_values)
-                    publisher.info('Url;{};{};{};Checked {} URL'.format(
+                    publisher.info('Url;{};{};{};Checked {} URL;{}'.format(
-                        PST.p_source, PST.p_date, PST.p_name, A_values[0]))
+                        PST.p_source, PST.p_date, PST.p_name, A_values[0], PST.p_path))
             prec_filename = filename
 
         else:

bin/WebStats.py

@@ -38,35 +38,55 @@ def get_date_range(num_day):
         date_list.append(date.substract_day(i))
     return date_list
 
+# Compute the progression for one keyword
+def compute_progression_word(server, num_day, keyword):
+    date_range = get_date_range(num_day)
+    # check if this keyword is eligible for progression
+    keyword_total_sum = 0
+    value_list = []
+    for date in date_range: # get value up to date_range
+        curr_value = server.hget(keyword, date)
+        value_list.append(int(curr_value if curr_value is not None else 0))
+        keyword_total_sum += int(curr_value) if curr_value is not None else 0
+    oldest_value = value_list[-1] if value_list[-1] != 0 else 1 #Avoid zero division
+
+    # The progression is based on the ratio: value[i] / value[i-1]
+    keyword_increase = 0
+    value_list_reversed = value_list[:]
+    value_list_reversed.reverse()
+    for i in range(1, len(value_list_reversed)):
+        divisor = value_list_reversed[i-1] if value_list_reversed[i-1] != 0 else 1
+        keyword_increase += value_list_reversed[i] / divisor
+
+    return (keyword_increase, keyword_total_sum)
+
+
+'''
+    recompute the set top_progression zset
+        - Compute the current field progression
+        - re-compute the current progression for each first 2*max_set_cardinality fields in the top_progression_zset
+'''
 def compute_progression(server, field_name, num_day, url_parsed):
-    redis_progression_name = 'top_progression_'+field_name
+    redis_progression_name_set = "z_top_progression_"+field_name
-    redis_progression_name_set = 'top_progression_'+field_name+'_set'
 
     keyword = url_parsed[field_name]
     if keyword is not None:
-        date_range = get_date_range(num_day) 
 
-        # check if this keyword is eligible for progression
+        #compute the progression of the current word
-        keyword_total_sum = 0 
+        keyword_increase, keyword_total_sum = compute_progression_word(server, num_day, keyword)
-        value_list = []
-        for date in date_range: # get value up to date_range
-            curr_value = server.hget(keyword, date)
-            value_list.append(int(curr_value if curr_value is not None else 0))
-            keyword_total_sum += int(curr_value) if curr_value is not None else 0
-        oldest_value = value_list[-1] if value_list[-1] != 0 else 1 #Avoid zero division
 
-        # The progression is based on the ratio: value[i] / value[i-1]
+        #re-compute the progression of 2*max_set_cardinality
-        keyword_increase = 0
+        current_top = server.zrevrangebyscore(redis_progression_name_set, '+inf', '-inf', withscores=True, start=0, num=2*max_set_cardinality)
-        value_list_reversed = value_list[:]
+        for word, value in current_top:
-        value_list_reversed.reverse()
+            word_inc, word_tot_sum = compute_progression_word(server, num_day, word)
-        for i in range(1, len(value_list_reversed)):
+            server.zrem(redis_progression_name_set, word)
-            divisor = value_list_reversed[i-1] if value_list_reversed[i-1] != 0 else 1
+            if (word_tot_sum > threshold_total_sum) and (word_inc > threshold_increase):
-            keyword_increase += value_list_reversed[i] / divisor
+                server.zadd(redis_progression_name_set, float(word_inc), word)
 
-        # filter
+        # filter before adding
         if (keyword_total_sum > threshold_total_sum) and (keyword_increase > threshold_increase):
+            server.zadd(redis_progression_name_set, float(keyword_increase), keyword)
 
-            server.zadd("z_top_progression_"+field_name, float(keyword_increase), keyword)
 
 
 if __name__ == '__main__':

bin/feeder/pystemon-feeder.py

@@ -24,13 +24,28 @@ import sys
 import time
 import redis
 import base64
+import os
+import ConfigParser
 
-port = "5556"
+configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
-pystemonpath = "/home/pystemon/pystemon/"
+if not os.path.exists(configfile):
+    raise Exception('Unable to find the configuration file. \
+        Did you set environment variables? \
+        Or activate the virtualenv.')
+
+cfg = ConfigParser.ConfigParser()
+cfg.read(configfile)
+
+if cfg.has_option("ZMQ_Global", "bind"):
+    zmq_url = cfg.get("ZMQ_Global", "bind")
+else:
+    zmq_url = "tcp://127.0.0.1:5556"
+
+pystemonpath = cfg.get("Directories", "pystemonpath")
 
 context = zmq.Context()
 socket = context.socket(zmq.PUB)
-socket.bind("tcp://*:%s" % port)
+socket.bind(zmq_url)
 
 # check https://github.com/cvandeplas/pystemon/blob/master/pystemon.yaml#L16
 r = redis.StrictRedis(host='localhost', db=10)

bin/packages/Paste.py

@@ -264,7 +264,7 @@ class Paste(object):
     
     def _get_p_duplicate(self):
         self.p_duplicate = self.store.hget(self.p_path, "p_duplicate")
-        return self.p_duplicate if self.p_duplicate is not None else []
+        return self.p_duplicate if self.p_duplicate is not None else '[]'
 
     def save_all_attributes_redis(self, key=None):
         """

bin/packages/config.cfg.sample

@@ -14,6 +14,10 @@ tldsfile = faup/src/data/mozilla.tlds
 
 domainstrending_csv = var/www/static/csv/domainstrendingdata
 
+pystemonpath = /home/pystemon/pystemon/
+
+sentiment_lexicon_file = sentiment/vader_lexicon.zip/vader_lexicon/vader_lexicon.txt
+
 ##### Flask #####
 [Flask]
 #Maximum number of character to display in the toolip
@@ -36,6 +40,15 @@ threshold_duplicate_tlsh = 100
 #Minimum size of the paste considered
 min_paste_size = 0.3
 
+[Module_ModuleInformation]
+#Threshold to deduce if a module is stuck or not, in seconds.
+threshold_stucked_module=600
+
+[Module_Mixer]
+#Define the configuration of the mixer, possible value: 1 or 2
+operation_mode = 1
+#Define the time that a paste will be considerate duplicate. in seconds (1day = 86400)
+ttl_duplicate = 86400
 
 ##### Redis #####
 [Redis_Cache]
@@ -63,6 +76,11 @@ host = localhost
 port = 6379
 db = 2
 
+[Redis_Mixer_Cache]
+host = localhost
+port = 6381
+db = 1
+
 ##### LevelDB #####
 [Redis_Level_DB_Curve]
 host = localhost
@@ -108,10 +126,13 @@ path = indexdir
 
 ###############################################################################
 
+# For multiple feed, add them with "," without space
+# e.g.: tcp://127.0.0.1:5556,tcp://127.0.0.1:5557
 [ZMQ_Global]
 #address = tcp://crf.circl.lu:5556
 address = tcp://127.0.0.1:5556
 channel = 102
+bind = tcp://127.0.0.1:5556
 
 [ZMQ_Url]
 address = tcp://127.0.0.1:5004

bin/packages/modules.cfg

@@ -1,7 +1,15 @@
-[Global]
+[Mixer]
 subscribe = ZMQ_Global
+publish = Redis_Mixer,Redis_preProcess1
+
+[Global]
+subscribe = Redis_Mixer
 publish = Redis_Global,Redis_ModuleStats
 
+[PreProcessFeed]
+subscribe = Redis_preProcess1
+publish = Redis_Mixer
+
 [Duplicates]
 subscribe = Redis_Duplicate
 
@@ -63,7 +71,7 @@ publish = Redis_BrowseWarningPaste,Redis_Duplicate
 [ModuleStats]
 subscribe = Redis_ModuleStats
 
-[Browse_warning_paste]
+[BrowseWarningPaste]
 subscribe = Redis_BrowseWarningPaste
 
 #[send_to_queue]
@@ -88,9 +96,6 @@ publish = Redis_BrowseWarningPaste,Redis_Duplicate
 subscribe = Redis_Global
 publish = Redis_Duplicate,Redis_BrowseWarningPaste
 
-[SourceCode]
-subscribe = Redis_SourceCode
-
 [Keys]
 subscribe = Redis_Global
 publish = Redis_Duplicate,Redis_BrowseWarningPaste

bin/preProcessFeed.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python2
+# -*-coding:UTF-8 -*
+
+import time
+from pubsublogger import publisher
+
+from Helper import Process
+
+
+def do_something(message):
+    splitted = message.split()
+    if len(splitted) == 2:
+        paste_name, gzip64encoded = splitted
+
+    paste_name = paste_name.replace("pastebin", "pastebinPROCESSED")
+
+    to_send = "{0} {1}".format(paste_name, gzip64encoded)
+    return to_send
+
+if __name__ == '__main__':
+    # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
+    # Port of the redis instance used by pubsublogger
+    publisher.port = 6380
+    # Script is the default channel used for the modules.
+    publisher.channel = 'Script'
+
+    # Section name in bin/packages/modules.cfg
+    config_section = 'PreProcessFeed'
+
+    # Setup the I/O queues
+    p = Process(config_section)
+
+    # Sent to the logging a description of the module
+    publisher.info("<description of the module>")
+
+    # Endless loop getting messages from the input queue
+    while True:
+        # Get one message from the input queue
+        message = p.get_from_set()
+        if message is None:
+            publisher.debug("{} queue is empty, waiting".format(config_section))
+            print "queue empty"
+            time.sleep(1)
+            continue
+
+        # Do something with the message from the queue
+        new_message = do_something(message)
+
+        # (Optional) Send that thing to the next queue
+        p.populate_set_out(new_message)

doc/all_modules.txt

@@ -0,0 +1,31 @@
+Attributes
+BrowseWarningPaste
+Categ
+Credential
+CreditCards
+Curve
+CurveManageTopSets
+Cve
+DomClassifier
+Duplicates
+Global
+Indexer
+Keys
+Lines
+Mail
+Mixer
+ModuleInformation
+Keys
+Lines
+Mail
+Mixer
+ModuleInformation
+ModuleStats
+Onion
+Phone
+Release
+SentimentAnalysis
+SQLInjectionDetection
+Tokenize
+Web
+WebStats

doc/generate_graph_data.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python2
 # -*-coding:UTF-8 -*
 
+import os
+
 content = ""
 modules = {}
 all_modules = []
@@ -8,7 +10,9 @@ curr_module = ""
 streamingPub = {}
 streamingSub = {}
 
-with open('../bin/packages/modules.cfg', 'r') as f:
+path = os.path.join(os.environ['AIL_BIN'], 'packages/modules.cfg')
+path2 = os.path.join(os.environ['AIL_HOME'], 'doc/all_modules.txt')
+with open(path, 'r') as f:
     for line in f:
         if line[0] != '#':
             if line[0] == '[':
@@ -32,7 +36,7 @@ with open('../bin/packages/modules.cfg', 'r') as f:
                     continue
 
     output_set_graph = set()
-    with open('all_modules.txt', 'w') as f2:
+    with open(path2, 'w') as f2:
         for e in all_modules:
             f2.write(e+"\n")
 

doc/generate_modules_data_flow_graph.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-python generate_graph_data.py | dot -T png -o module-data-flow.png
+python $AIL_HOME/doc/generate_graph_data.py | dot -T png -o $AIL_HOME/doc/module-data-flow.png

docker_start.sh

@@ -0,0 +1,75 @@
+source ./AILENV/bin/activate
+cd bin
+
+export PATH=$AIL_HOME:$PATH
+export PATH=$AIL_REDIS:$PATH
+export PATH=$AIL_LEVELDB:$PATH
+export AILENV=/opt/AIL
+
+conf_dir="${AIL_HOME}/configs/"
+
+screen -dmS "Redis"
+screen -S "Redis" -X screen -t "6379" bash -c 'redis-server '$conf_dir'6379.conf ; read x'
+screen -S "Redis" -X screen -t "6380" bash -c 'redis-server '$conf_dir'6380.conf ; read x'
+screen -S "Redis" -X screen -t "6381" bash -c 'redis-server '$conf_dir'6381.conf ; read x'
+
+# For Words and curves
+sleep 0.1
+screen -S "Redis" -X screen -t "6382" bash -c 'redis-server '$conf_dir'6382.conf ; read x'
+
+#Want to launch more level_db?
+lvdbhost='127.0.0.1'
+lvdbdir="${AIL_HOME}/LEVEL_DB_DATA/"
+db1_y='2013'
+db2_y='2014'
+db3_y='2016'
+db4_y='3016'
+nb_db=13
+
+screen -dmS "LevelDB"
+#Add lines here with appropriates options.
+screen -S "LevelDB" -X screen -t "2013" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2013/ -P '$db1_y' -M '$nb_db'; read x'
+screen -S "LevelDB" -X screen -t "2014" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2014/ -P '$db2_y' -M '$nb_db'; read x'
+screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db3_y' -M '$nb_db'; read x'
+
+# For Curve
+screen -S "LevelDB" -X screen -t "3016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'3016/ -P '$db4_y' -M '$nb_db'; read x'
+
+
+screen -dmS "Logging"
+screen -S "Logging" -X screen -t "LogQueue" bash -c 'log_subscriber -p 6380 -c Queuing -l ../logs/; read x'
+screen -S "Logging" -X screen -t "LogScript" bash -c 'log_subscriber -p 6380 -c Script -l ../logs/; read x'
+
+screen -dmS "Queue"
+screen -S "Queue" -X screen -t "Queues" bash -c './launch_queues.py; read x'
+
+screen -dmS "Script"
+screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModuleInformation.py -k 0 -c 1; read x'
+screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x'
+screen -S "Script" -X screen -t "Duplicates" bash -c './Duplicates.py; read x'
+screen -S "Script" -X screen -t "Attributes" bash -c './Attributes.py; read x'
+screen -S "Script" -X screen -t "Lines" bash -c './Lines.py; read x'
+screen -S "Script" -X screen -t "DomClassifier" bash -c './DomClassifier.py; read x'
+screen -S "Script" -X screen -t "Categ" bash -c './Categ.py; read x'
+screen -S "Script" -X screen -t "Tokenize" bash -c './Tokenize.py; read x'
+screen -S "Script" -X screen -t "CreditCards" bash -c './CreditCards.py; read x'
+screen -S "Script" -X screen -t "Onion" bash -c './Onion.py; read x'
+screen -S "Script" -X screen -t "Mail" bash -c './Mail.py; read x'
+screen -S "Script" -X screen -t "Web" bash -c './Web.py; read x'
+screen -S "Script" -X screen -t "Credential" bash -c './Credential.py; read x'
+screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x'
+screen -S "Script" -X screen -t "CurveManageTopSets" bash -c './CurveManageTopSets.py; read x'
+screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x'
+screen -S "Script" -X screen -t "Keys" bash -c './Keys.py; read x'
+screen -S "Script" -X screen -t "Phone" bash -c './Phone.py; read x'
+screen -S "Script" -X screen -t "Release" bash -c './Release.py; read x'
+screen -S "Script" -X screen -t "Cve" bash -c './Cve.py; read x'
+screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x'
+screen -S "Script" -X screen -t "ModuleStats" bash -c './ModuleStats.py; read x'
+screen -S "Script" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x'
+screen -S "Script" -X screen -t "BrowseWarningPaste" bash -c './BrowseWarningPaste.py; read x'
+screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x'
+
+cd $AILENV
+cd var/www/
+python Flask_server.py

installing_deps.sh

@@ -6,23 +6,25 @@ set -x
 sudo apt-get update
 
 sudo apt-get install python-pip python-virtualenv python-dev libfreetype6-dev \
-    screen g++ python-tk unzip libsnappy-dev cmake
+    screen g++ python-tk unzip libsnappy-dev cmake -y
 
 #Needed for bloom filters
-sudo apt-get install libssl-dev libfreetype6-dev python-numpy
+sudo apt-get install libssl-dev libfreetype6-dev python-numpy -y
 
 # DNS deps
-sudo apt-get install libadns1 libadns1-dev
+sudo apt-get install libadns1 libadns1-dev -y
 
 #Needed for redis-lvlDB
-sudo apt-get install libev-dev libgmp-dev
+sudo apt-get install libev-dev libgmp-dev -y
 
 #Need for generate-data-flow graph
-sudo apt-get install graphviz
+sudo apt-get install graphviz -y
 
 #needed for mathplotlib
-test ! -L /usr/include/ft2build.h && sudo ln -s freetype2/ft2build.h /usr/include/
 sudo easy_install -U distribute
+# ssdeep
+sudo apt-get install libfuzzy-dev
+sudo apt-get install build-essential libffi-dev automake autoconf libtool -y
 
 # REDIS #
 test ! -d redis/ && git clone https://github.com/antirez/redis.git
@@ -32,7 +34,7 @@ make
 popd
 
 # Faup
-test ! -d faup && git clone https://github.com/stricaud/faup.git
+test ! -d faup/ && git clone https://github.com/stricaud/faup.git
 pushd faup/
 test ! -d build && mkdir build
 cd build
@@ -46,6 +48,10 @@ popd
 test ! -d tlsh && git clone git://github.com/trendmicro/tlsh.git
 pushd tlsh/
 ./make.sh
+pushd build/release/
+sudo make install
+sudo ldconfig
+popd
 popd
 
 # REDIS LEVEL DB #
@@ -60,22 +66,30 @@ if [ ! -f bin/packages/config.cfg ]; then
     cp bin/packages/config.cfg.sample bin/packages/config.cfg
 fi
 
-virtualenv AILENV
+pushd var/www/
+./update_thirdparty.sh
+popd
 
-echo export AIL_HOME=$(pwd) >> ./AILENV/bin/activate
+if [ -z "$VIRTUAL_ENV" ]; then
-echo export AIL_BIN=$(pwd)/bin/ >> ./AILENV/bin/activate
-echo export AIL_FLASK=$(pwd)/var/www/ >> ./AILENV/bin/activate
-echo export AIL_REDIS=$(pwd)/redis/src/ >> ./AILENV/bin/activate
-echo export AIL_LEVELDB=$(pwd)/redis-leveldb/ >> ./AILENV/bin/activate
 
-. ./AILENV/bin/activate
+    virtualenv AILENV
+
+    echo export AIL_HOME=$(pwd) >> ./AILENV/bin/activate
+    echo export AIL_BIN=$(pwd)/bin/ >> ./AILENV/bin/activate
+    echo export AIL_FLASK=$(pwd)/var/www/ >> ./AILENV/bin/activate
+    echo export AIL_REDIS=$(pwd)/redis/src/ >> ./AILENV/bin/activate
+    echo export AIL_LEVELDB=$(pwd)/redis-leveldb/ >> ./AILENV/bin/activate
+
+    . ./AILENV/bin/activate
+
+fi
 
 mkdir -p $AIL_HOME/{PASTES,Blooms,dumps}
 mkdir -p $AIL_HOME/LEVEL_DB_DATA/2016
 mkdir -p $AIL_HOME/LEVEL_DB_DATA/3016
 
 pip install -U pip
-pip install -r pip_packages_requirement.txt
+pip install -U -r pip_packages_requirement.txt
 
 # Pyfaup
 pushd faup/src/lib/bindings/python/
@@ -90,3 +104,7 @@ python setup.py install
 # Download the necessary NLTK corpora and sentiment vader
 HOME=$(pwd) python -m textblob.download_corpora
 python -m nltk.downloader vader_lexicon
+python -m nltk.downloader punkt
+
+#Create the file all_module and update the graph in doc
+$AIL_HOME/doc/generate_modules_data_flow_graph.sh

pip_packages_requirement.txt

@@ -11,6 +11,7 @@ numpy
 matplotlib
 networkx
 terminaltables
+colorama
 
 #Tokeniser
 nltk
@@ -40,7 +41,7 @@ pycountry
 PySocks
 
 #ASN lookup requirements
-http://adns-python.googlecode.com/files/adns-python-1.2.1.tar.gz
+https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/adns-python/adns-python-1.2.1.tar.gz
 https://github.com/trolldbois/python-cymru-services/archive/master.zip
 
 https://github.com/saffsd/langid.py/archive/master.zip

var/www/Flask_server.py

@@ -12,209 +12,34 @@ import flask
 import os
 import sys
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
+sys.path.append('./Flasks/')
 import Paste
 from Date import Date
 
+# Import config
+import Flask_config
+
 # CONFIG #
-tlsh_to_percent = 1000.0 #Use to display the estimated percentage instead of a raw value
+cfg = Flask_config.cfg
 
-configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
+Flask_config.app = Flask(__name__, static_url_path='/static/')
-if not os.path.exists(configfile):
+app = Flask_config.app
-    raise Exception('Unable to find the configuration file. \
-                    Did you set environment variables? \
-                    Or activate the virtualenv.')
-
-cfg = ConfigParser.ConfigParser()
-cfg.read(configfile)
-
-max_preview_char = int(cfg.get("Flask", "max_preview_char")) # Maximum number of character to display in the tooltip
-max_preview_modal = int(cfg.get("Flask", "max_preview_modal")) # Maximum number of character to display in the modal
-
-# REDIS #
-r_serv = redis.StrictRedis(
-    host=cfg.get("Redis_Queues", "host"),
-    port=cfg.getint("Redis_Queues", "port"),
-    db=cfg.getint("Redis_Queues", "db"))
-
-r_serv_log = redis.StrictRedis(
-    host=cfg.get("Redis_Log", "host"),
-    port=cfg.getint("Redis_Log", "port"),
-    db=cfg.getint("Redis_Log", "db"))
-
-r_serv_charts = redis.StrictRedis(
-    host=cfg.get("Redis_Level_DB_Trending", "host"),
-    port=cfg.getint("Redis_Level_DB_Trending", "port"),
-    db=cfg.getint("Redis_Level_DB_Trending", "db"))
-
-r_serv_db = redis.StrictRedis(
-    host=cfg.get("Redis_Level_DB", "host"),
-    port=cfg.getint("Redis_Level_DB", "port"),
-    db=cfg.getint("Redis_Level_DB", "db"))
-
-r_serv_sentiment = redis.StrictRedis(
-        host=cfg.get("Redis_Level_DB_Sentiment", "host"),
-        port=cfg.getint("Redis_Level_DB_Sentiment", "port"),
-        db=cfg.getint("Redis_Level_DB_Sentiment", "db"))
-
-r_serv_term = redis.StrictRedis(
-        host=cfg.get("Redis_Level_DB_TermFreq", "host"),
-        port=cfg.getint("Redis_Level_DB_TermFreq", "port"),
-        db=cfg.getint("Redis_Level_DB_TermFreq", "db"))
-
-r_serv_pasteName = redis.StrictRedis(
-    host=cfg.get("Redis_Paste_Name", "host"),
-    port=cfg.getint("Redis_Paste_Name", "port"),
-    db=cfg.getint("Redis_Paste_Name", "db"))
-
-
-app = Flask(__name__, static_url_path='/static/')
-
-
-def event_stream():
-    pubsub = r_serv_log.pubsub()
-    pubsub.psubscribe("Script" + '.*')
-    for msg in pubsub.listen():
-        level = msg['channel'].split('.')[1]
-        if msg['type'] == 'pmessage' and level != "DEBUG":
-            yield 'data: %s\n\n' % json.dumps(msg)
-
-
-def get_queues(r):
-    # We may want to put the llen in a pipeline to do only one query.
-    data = [(queue, int(card)) for queue, card in r.hgetall("queues").iteritems()]
-    newData = []
-
-    curr_range = 50
-    for queue, card in data:
-        key = "MODULE_" + queue + "_"
-        for i in range(1, 50):
-            curr_num = r.get("MODULE_"+ queue + "_" + str(i))
-            if curr_num is None:
-                curr_range = i
-                break
-
-        for moduleNum in range(1, curr_range):
-            value = r.get(key + str(moduleNum))
-            if value is not None:
-                timestamp, path = value.split(", ")
-                if timestamp is not None:
-                    startTime_readable = datetime.datetime.fromtimestamp(int(timestamp))
-                    processed_time_readable = str((datetime.datetime.now() - startTime_readable)).split('.')[0]
-                    seconds = int((datetime.datetime.now() - startTime_readable).total_seconds())
-                    newData.append( (queue, card, seconds, moduleNum) )
-                else:
-                    newData.append( (queue, cards, 0, moduleNum) )
-
-    return newData
 
+# import routes and functions from modules
+import Flask_dashboard
+import Flask_trendingcharts
+import Flask_trendingmodules
+import Flask_browsepastes
+import Flask_sentiment
+import Flask_terms
+import Flask_search
+import Flask_showpaste
 
 def list_len(s):
     return len(s)
 app.jinja_env.filters['list_len'] = list_len
 
 
-def showpaste(content_range):
-    requested_path = request.args.get('paste', '')
-    paste = Paste.Paste(requested_path)
-    p_date = str(paste._get_p_date())
-    p_date = p_date[6:]+'/'+p_date[4:6]+'/'+p_date[0:4]
-    p_source = paste.p_source
-    p_encoding = paste._get_p_encoding()
-    p_language = paste._get_p_language()
-    p_size = paste.p_size
-    p_mime = paste.p_mime
-    p_lineinfo = paste.get_lines_info()
-    p_content = paste.get_p_content().decode('utf-8', 'ignore')
-    p_duplicate_full_list = json.loads(paste._get_p_duplicate())
-    p_duplicate_list = []
-    p_simil_list = []
-    p_hashtype_list = []
-
-
-    for dup_list in p_duplicate_full_list:
-        if dup_list[0] == "tlsh":
-            dup_list[2] = int(((tlsh_to_percent - float(dup_list[2])) / tlsh_to_percent)*100)
-        else:
-            dup_list[2] = int(dup_list[2])
-
-    p_duplicate_full_list.sort(lambda x,y: cmp(x[2], y[2]), reverse=True)
-
-    # Combine multiple duplicate paste name and format for display
-    new_dup_list = []
-    dup_list_removed = []
-    for dup_list_index in range(0, len(p_duplicate_full_list)):
-        if dup_list_index in dup_list_removed:
-            continue
-        indices = [i for i, x in enumerate(p_duplicate_full_list) if x[1] == p_duplicate_full_list[dup_list_index][1]]
-        hash_types = []
-        comp_vals = []
-        for i in indices:
-            hash_types.append(p_duplicate_full_list[i][0].encode('utf8'))
-            comp_vals.append(p_duplicate_full_list[i][2])
-            dup_list_removed.append(i)
-
-        hash_types = str(hash_types).replace("[","").replace("]","") if len(hash_types)==1 else str(hash_types)
-        comp_vals = str(comp_vals).replace("[","").replace("]","") if len(comp_vals)==1 else str(comp_vals)
-        new_dup_list.append([hash_types.replace("'", ""), p_duplicate_full_list[dup_list_index][1], comp_vals])
-
-    # Create the list to pass to the webpage
-    for dup_list in new_dup_list:
-        hash_type, path, simil_percent = dup_list
-        p_duplicate_list.append(path)
-        p_simil_list.append(simil_percent)
-        p_hashtype_list.append(hash_type)
-
-    if content_range != 0:
-       p_content = p_content[0:content_range]
-
-
-    return render_template("show_saved_paste.html", date=p_date, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=len(p_content), duplicate_list = p_duplicate_list, simil_list = p_simil_list, hashtype_list = p_hashtype_list)
-
-def getPastebyType(server, module_name):
-    all_path = []
-    for path in server.smembers('WARNING_'+module_name):
-        all_path.append(path)
-    return all_path
-
-
-def get_date_range(num_day):
-    curr_date = datetime.date.today()
-    date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2))
-    date_list = []
-
-    for i in range(0, num_day+1):
-        date_list.append(date.substract_day(i))
-    return date_list
-
-# Iterate over elements in the module provided and return the today data or the last data
-# return format: [('passed_days', num_of_passed_days), ('elem_name1', elem_value1), ('elem_name2', elem_value2)]]
-def get_top_relevant_data(server, module_name):
-    days = 0
-    for date in get_date_range(15):
-        redis_progression_name_set = 'top_'+ module_name +'_set_' + date
-        member_set = server.zrevrangebyscore(redis_progression_name_set, '+inf', '-inf', withscores=True)
-        if len(member_set) == 0: #No data for this date
-            days += 1
-        else:
-            member_set.insert(0, ("passed_days", days))
-            return member_set
-
-
-def Term_getValueOverRange(word, startDate, num_day):
-    passed_days = 0
-    oneDay = 60*60*24
-    to_return = []
-    curr_to_return = 0
-    for timestamp in range(startDate, startDate - max(num_day)*oneDay, -oneDay):
-        value = r_serv_term.hget(timestamp, word)
-        curr_to_return += int(value) if value is not None else 0
-        for i in num_day:
-            if passed_days == i-1:
-                to_return.append(curr_to_return)
-        passed_days += 1
-    return to_return
-
-
 # ========= CACHE CONTROL ========
 @app.after_request
 def add_header(response):
@@ -226,533 +51,7 @@ def add_header(response):
     response.headers['Cache-Control'] = 'public, max-age=0'
     return response
 
-# ============ ROUTES ============
+# ============ MAIN ============
-
-@app.route("/_logs")
-def logs():
-    return flask.Response(event_stream(), mimetype="text/event-stream")
-
-
-@app.route("/_stuff", methods=['GET'])
-def stuff():
-    return jsonify(row1=get_queues(r_serv))
-
-@app.route("/_progressionCharts", methods=['GET'])
-def progressionCharts():
-    attribute_name = request.args.get('attributeName')
-    trending_name = request.args.get('trendingName')
-    bar_requested = True if request.args.get('bar') == "true" else False
-
-    if (bar_requested):
-        num_day = int(request.args.get('days'))
-        bar_values = []
-
-        date_range = get_date_range(num_day)
-        # Retreive all data from the last num_day
-        for date in date_range:
-            curr_value = r_serv_charts.hget(attribute_name, date)
-            bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], int(curr_value if curr_value is not None else 0)])
-        bar_values.insert(0, attribute_name)
-        return jsonify(bar_values)
-
-    else:
-        redis_progression_name = "z_top_progression_" + trending_name
-        keyw_value = r_serv_charts.zrevrangebyscore(redis_progression_name, '+inf', '-inf', withscores=True, start=0, num=10)
-        return jsonify(keyw_value)
-
-@app.route("/_moduleCharts", methods=['GET'])
-def modulesCharts():
-    keyword_name = request.args.get('keywordName')
-    module_name = request.args.get('moduleName')
-    bar_requested = True if request.args.get('bar') == "true" else False
-
-    if (bar_requested):
-        num_day = int(request.args.get('days'))
-        bar_values = []
-
-        date_range = get_date_range(num_day)
-        # Retreive all data from the last num_day
-        for date in date_range:
-            curr_value = r_serv_charts.hget(date, module_name+'-'+keyword_name)
-            bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], int(curr_value if curr_value is not None else 0)])
-        bar_values.insert(0, keyword_name)
-        return jsonify(bar_values)
-
-    else:
-        member_set = get_top_relevant_data(r_serv_charts, module_name)
-        if len(member_set) == 0:
-            member_set.append(("No relevant data", int(100)))
-        return jsonify(member_set)
-
-
-@app.route("/_providersChart", methods=['GET'])
-def providersChart():
-    keyword_name = request.args.get('keywordName')
-    module_name = request.args.get('moduleName')
-    bar_requested = True if request.args.get('bar') == "true" else False
-
-    if (bar_requested):
-        num_day = int(request.args.get('days'))
-        bar_values = []
-
-        date_range = get_date_range(num_day)
-        # Retreive all data from the last num_day
-        for date in date_range:
-            curr_value_size = r_serv_charts.hget(keyword_name+'_'+'size', date)
-            curr_value_num = r_serv_charts.hget(keyword_name+'_'+'num', date)
-            curr_value_size_avg = r_serv_charts.hget(keyword_name+'_'+'avg', date)
-            if module_name == "size":
-                curr_value = float(curr_value_size_avg if curr_value_size_avg is not None else 0)
-            else:
-                curr_value = float(curr_value_num if curr_value_num is not None else 0.0)
-
-            bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], curr_value])
-        bar_values.insert(0, keyword_name)
-        return jsonify(bar_values)
-
-    else:
-        #redis_provider_name_set = 'top_size_set' if module_name == "size" else 'providers_set'
-        redis_provider_name_set = 'top_avg_size_set_' if module_name == "size" else 'providers_set_'
-        redis_provider_name_set = redis_provider_name_set + get_date_range(0)[0]
-        
-        member_set = r_serv_charts.zrevrangebyscore(redis_provider_name_set, '+inf', '-inf', withscores=True, start=0, num=8)
-        # Member set is a list of (value, score) pairs
-        if len(member_set) == 0:
-            member_set.append(("No relevant data", float(100)))
-        return jsonify(member_set)
-
-
-
-@app.route("/search", methods=['POST'])
-def search():
-    query = request.form['query']
-    q = []
-    q.append(query)
-    r = [] #complete path
-    c = [] #preview of the paste content
-    paste_date = []
-    paste_size = []
-
-    # Search filename
-    print r_serv_pasteName.smembers(q[0])
-    for path in r_serv_pasteName.smembers(q[0]):
-        print path
-        r.append(path)
-        paste = Paste.Paste(path)
-        content = paste.get_p_content().decode('utf8', 'ignore')
-        content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
-        c.append(content[0:content_range])
-        curr_date = str(paste._get_p_date())
-        curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
-        paste_date.append(curr_date)
-        paste_size.append(paste._get_p_size())
-
-    # Search full line
-    from whoosh import index
-    from whoosh.fields import Schema, TEXT, ID
-    schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT)
-
-    indexpath = os.path.join(os.environ['AIL_HOME'], cfg.get("Indexer", "path"))
-    ix = index.open_dir(indexpath)
-    from whoosh.qparser import QueryParser
-    with ix.searcher() as searcher:
-        query = QueryParser("content", ix.schema).parse(" ".join(q))
-        results = searcher.search(query, limit=None)
-        for x in results:
-            r.append(x.items()[0][1])
-            paste = Paste.Paste(x.items()[0][1])
-            content = paste.get_p_content().decode('utf8', 'ignore')
-            content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
-            c.append(content[0:content_range])
-            curr_date = str(paste._get_p_date())
-            curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
-            paste_date.append(curr_date)
-            paste_size.append(paste._get_p_size())
-    return render_template("search.html", r=r, c=c, query=request.form['query'], paste_date=paste_date, paste_size=paste_size, char_to_display=max_preview_modal)
-
-
-@app.route("/")
-def index():
-    default_minute = cfg.get("Flask", "minute_processed_paste")
-    return render_template("index.html", default_minute = default_minute)
-
-
-@app.route("/monitoring/")
-def monitoring():
-    for queue in r_serv.smembers("queues"):
-        return render_template("Queue_live_Monitoring.html", last_value=queue)
-
-
-@app.route("/wordstrending/")
-def wordstrending():
-    default_display = cfg.get("Flask", "default_display")
-    return render_template("Wordstrending.html", default_display = default_display)
-
-
-@app.route("/protocolstrending/")
-def protocolstrending():
-    default_display = cfg.get("Flask", "default_display")
-    return render_template("Protocolstrending.html", default_display = default_display)
-
-
-@app.route("/trending/")
-def trending():
-    default_display = cfg.get("Flask", "default_display")
-    return render_template("Trending.html", default_display = default_display)
-
-@app.route("/browseImportantPaste/", methods=['GET'])
-def browseImportantPaste():
-    module_name = request.args.get('moduleName')
-    return render_template("browse_important_paste.html")
-
-
-@app.route("/importantPasteByModule/", methods=['GET'])
-def importantPasteByModule():
-    module_name = request.args.get('moduleName')
-
-    all_content = []
-    paste_date = []
-    paste_linenum = []
-    all_path = []
-
-    for path in getPastebyType(r_serv_db, module_name):
-        all_path.append(path)
-        paste = Paste.Paste(path)
-        content = paste.get_p_content().decode('utf8', 'ignore')
-        content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
-        all_content.append(content[0:content_range])
-        curr_date = str(paste._get_p_date())
-        curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
-        paste_date.append(curr_date)
-        paste_linenum.append(paste.get_lines_info()[0])
-
-    return render_template("important_paste_by_module.html", all_path=all_path, content=all_content, paste_date=paste_date, paste_linenum=paste_linenum, char_to_display=max_preview_modal)
-
-@app.route("/moduletrending/")
-def moduletrending():
-    return render_template("Moduletrending.html")
-
-@app.route("/sentiment_analysis_trending/")
-def sentiment_analysis_trending():
-    return render_template("sentiment_analysis_trending.html")
-
-
-@app.route("/sentiment_analysis_getplotdata/", methods=['GET'])
-def sentiment_analysis_getplotdata():
-    # Get the top providers based on number of pastes
-    oneHour = 60*60
-    sevenDays = oneHour*24*7
-    dateStart = datetime.datetime.now()
-    dateStart = dateStart.replace(minute=0, second=0, microsecond=0)
-    dateStart_timestamp = calendar.timegm(dateStart.timetuple())
-
-    getAllProviders = request.args.get('getProviders')
-    provider = request.args.get('provider')
-    allProvider = request.args.get('all')
-    if getAllProviders == 'True':
-        if allProvider == "True":
-            range_providers = r_serv_charts.smembers('all_provider_set')
-            return jsonify(list(range_providers))
-        else:
-            range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(0)[0], '+inf', '-inf', start=0, num=8)
-            # if empty, get yesterday top providers
-            range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(1)[1], '+inf', '-inf', start=0, num=8) if range_providers == [] else range_providers
-            # if still empty, takes from all providers
-            if range_providers == []:
-                print 'today provider empty'
-                range_providers = r_serv_charts.smembers('all_provider_set')
-            return jsonify(range_providers)
-
-    elif provider is not None:
-        to_return = {}
-
-        cur_provider_name = provider + '_'
-        list_date = {}
-        for cur_timestamp in range(int(dateStart_timestamp), int(dateStart_timestamp)-sevenDays-oneHour, -oneHour):
-            cur_set_name = cur_provider_name + str(cur_timestamp)
-
-            list_value = []
-            for cur_id in r_serv_sentiment.smembers(cur_set_name):
-                cur_value = r_serv_sentiment.get(cur_id)
-                list_value.append(cur_value)
-            list_date[cur_timestamp] = list_value
-        to_return[provider] = list_date
-
-        return jsonify(to_return)
-    return "Bad request"
-
-
-
-@app.route("/sentiment_analysis_plot_tool/")
-def sentiment_analysis_plot_tool():
-    return render_template("sentiment_analysis_plot_tool.html")
-
-
-
-@app.route("/sentiment_analysis_plot_tool_getdata/", methods=['GET'])
-def sentiment_analysis_plot_tool_getdata():
-    getProviders = request.args.get('getProviders')
-
-    if getProviders == 'True':
-        providers = []
-        for cur_provider in r_serv_charts.smembers('all_provider_set'):
-            providers.append(cur_provider)
-        return jsonify(providers)
-
-    else:
-        query = request.args.get('query')
-        query = query.split(',')
-        Qdate = request.args.get('Qdate')
-
-        date1 = (Qdate.split('-')[0]).split('.')
-        date1 = datetime.date(int(date1[2]), int(date1[1]), int(date1[0]))
-
-        date2 = (Qdate.split('-')[1]).split('.')
-        date2 = datetime.date(int(date2[2]), int(date2[1]), int(date2[0]))
-
-        timestamp1 = calendar.timegm(date1.timetuple())
-        timestamp2 = calendar.timegm(date2.timetuple())
-
-        oneHour = 60*60
-        oneDay = oneHour*24
-
-        to_return = {}
-        for cur_provider in query:
-            list_date = {}
-            cur_provider_name = cur_provider + '_'
-            for cur_timestamp in range(int(timestamp1), int(timestamp2)+oneDay, oneHour):
-                cur_set_name = cur_provider_name + str(cur_timestamp)
-
-                list_value = []
-                for cur_id in r_serv_sentiment.smembers(cur_set_name):
-                    cur_value = r_serv_sentiment.get(cur_id)
-                    list_value.append(cur_value)
-                list_date[cur_timestamp] = list_value
-            to_return[cur_provider] = list_date
-
-        return jsonify(to_return)
-
-
-@app.route("/terms_management/")
-def terms_management():
-    TrackedTermsSet_Name = "TrackedSetTermSet"
-    BlackListTermsSet_Name = "BlackListSetTermSet"
-    TrackedTermsDate_Name = "TrackedTermDate"
-    BlackListTermsDate_Name = "BlackListTermDate"
-    
-    today = datetime.datetime.now()
-    today = today.replace(hour=0, minute=0, second=0, microsecond=0)
-    today_timestamp = calendar.timegm(today.timetuple())
-
-    track_list = []
-    track_list_values = []
-    track_list_num_of_paste = []
-    for tracked_term in r_serv_term.smembers(TrackedTermsSet_Name):
-        track_list.append(tracked_term)
-        value_range = Term_getValueOverRange(tracked_term, today_timestamp, [1, 7, 31])
-
-        term_date = r_serv_term.hget(TrackedTermsDate_Name, tracked_term)
-
-        set_paste_name = "tracked_" + tracked_term
-        track_list_num_of_paste.append(r_serv_term.scard(set_paste_name))
-        term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded"
-        value_range.append(term_date)
-        track_list_values.append(value_range)
-
-
-    black_list = []
-    for blacked_term in r_serv_term.smembers(BlackListTermsSet_Name):
-        term_date = r_serv_term.hget(BlackListTermsDate_Name, blacked_term)
-        term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded"
-        black_list.append([blacked_term, term_date])
-
-    return render_template("terms_management.html", black_list=black_list, track_list=track_list, track_list_values=track_list_values,  track_list_num_of_paste=track_list_num_of_paste)
-
-
-@app.route("/terms_management_query_paste/")
-def terms_management_query_paste():
-    term =  request.args.get('term')
-    TrackedTermsSet_Name = "TrackedSetTermSet"
-    paste_info = []
-
-    set_paste_name = "tracked_" + term
-    track_list_path = r_serv_term.smembers(set_paste_name)
-
-    for path in track_list_path:
-        paste = Paste.Paste(path)
-        p_date = str(paste._get_p_date())
-        p_date = p_date[6:]+'/'+p_date[4:6]+'/'+p_date[0:4]
-        p_source = paste.p_source
-        p_encoding = paste._get_p_encoding()
-        p_size = paste.p_size
-        p_mime = paste.p_mime
-        p_lineinfo = paste.get_lines_info()
-        p_content = paste.get_p_content().decode('utf-8', 'ignore')
-        if p_content != 0:
-            p_content = p_content[0:400]
-        paste_info.append({"path": path, "date": p_date, "source": p_source, "encoding": p_encoding, "size": p_size, "mime": p_mime, "lineinfo": p_lineinfo, "content": p_content})
-
-    return jsonify(paste_info)
-
-
-@app.route("/terms_management_query/")
-def terms_management_query():
-    TrackedTermsDate_Name = "TrackedTermDate"
-    BlackListTermsDate_Name = "BlackListTermDate"
-    term =  request.args.get('term')
-    section = request.args.get('section')
-
-    today = datetime.datetime.now()
-    today = today.replace(hour=0, minute=0, second=0, microsecond=0)
-    today_timestamp = calendar.timegm(today.timetuple())
-    value_range = Term_getValueOverRange(term, today_timestamp, [1, 7, 31])
-
-    if section == "followTerm":
-        term_date = r_serv_term.hget(TrackedTermsDate_Name, term)
-    elif section == "blacklistTerm":
-        term_date = r_serv_term.hget(BlackListTermsDate_Name, term)
-
-    term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded"
-    value_range.append(str(term_date))
-    return jsonify(value_range)
-
-
-@app.route("/terms_management_action/", methods=['GET'])
-def terms_management_action():
-    TrackedTermsSet_Name = "TrackedSetTermSet"
-    TrackedTermsDate_Name = "TrackedTermDate"
-    BlackListTermsDate_Name = "BlackListTermDate"
-    BlackListTermsSet_Name = "BlackListSetTermSet"
-
-    today = datetime.datetime.now()
-    today = today.replace(microsecond=0)
-    today_timestamp = calendar.timegm(today.timetuple())
-
-
-    section = request.args.get('section')
-    action = request.args.get('action')
-    term =  request.args.get('term')
-    if action is None or term is None:
-        return "None"
-    else:
-        if section == "followTerm":
-            if action == "add":
-                r_serv_term.sadd(TrackedTermsSet_Name, term.lower())
-                r_serv_term.hset(TrackedTermsDate_Name, term, today_timestamp)
-            else:
-                r_serv_term.srem(TrackedTermsSet_Name, term.lower())
-        elif section == "blacklistTerm":
-            if action == "add":
-                r_serv_term.sadd(BlackListTermsSet_Name, term.lower())
-                r_serv_term.hset(BlackListTermsDate_Name, term, today_timestamp)
-            else:
-                r_serv_term.srem(BlackListTermsSet_Name, term.lower())
-        else:
-            return "None"
-
-        to_return = {}
-        to_return["section"] = section
-        to_return["action"] = action
-        to_return["term"] = term
-        return jsonify(to_return)
-
-
-
-@app.route("/terms_plot_tool/")
-def terms_plot_tool():
-    term =  request.args.get('term')
-    if term is not None:
-        return render_template("terms_plot_tool.html", term=term)
-    else:
-        return render_template("terms_plot_tool.html", term="")
-
-
-@app.route("/terms_plot_tool_data/")
-def terms_plot_tool_data():
-    oneDay = 60*60*24
-    range_start =  datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_start')))) if request.args.get('range_start') is not None else 0;
-    range_start = range_start.replace(hour=0, minute=0, second=0, microsecond=0)
-    range_start = calendar.timegm(range_start.timetuple())
-    range_end =  datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_end')))) if request.args.get('range_end') is not None else 0;
-    range_end = range_end.replace(hour=0, minute=0, second=0, microsecond=0)
-    range_end = calendar.timegm(range_end.timetuple())
-    term =  request.args.get('term')
-
-    if term is None:
-        return "None"
-    else:
-        value_range = []
-        for timestamp in range(range_start, range_end+oneDay, oneDay):
-            print timestamp, term
-            value = r_serv_term.hget(timestamp, term)
-            curr_value_range = int(value) if value is not None else 0
-            value_range.append([timestamp, curr_value_range])
-        return jsonify(value_range)
-
-
-@app.route("/terms_plot_top/")
-def terms_plot_top():
-    return render_template("terms_plot_top.html")
-
-
-@app.route("/terms_plot_top_data/")
-def terms_plot_top_data():
-    oneDay = 60*60*24
-    today = datetime.datetime.now()
-    today = today.replace(hour=0, minute=0, second=0, microsecond=0)
-    today_timestamp = calendar.timegm(today.timetuple())
-
-    set_day = "TopTermFreq_set_day_" + str(today_timestamp)
-    set_week = "TopTermFreq_set_week";
-    set_month = "TopTermFreq_set_month";
-
-    the_set = request.args.get('set')
-    num_day = int(request.args.get('num_day'))
-    if the_set is None:
-        return "None"
-    else:
-        to_return = []
-        if the_set == "TopTermFreq_set_day":
-            the_set += "_" + str(today_timestamp)
-
-        for term, tot_value in r_serv_term.zrevrangebyscore(the_set, '+inf', '-inf', withscores=True, start=0, num=20):
-            position = {}
-            position['day'] = r_serv_term.zrevrank(set_day, term)
-            position['day'] = position['day']+1 if position['day'] is not None else "<20"
-            position['week'] = r_serv_term.zrevrank(set_week, term)
-            position['week'] = position['week']+1 if position['week'] is not None else "<20"
-            position['month'] = r_serv_term.zrevrank(set_month, term)
-            position['month'] = position['month']+1 if position['month'] is not None else "<20"
-            value_range = []
-            for timestamp in range(today_timestamp, today_timestamp - num_day*oneDay, -oneDay):
-                value = r_serv_term.hget(timestamp, term)
-                curr_value_range = int(value) if value is not None else 0
-                value_range.append([timestamp, curr_value_range])
-                
-            to_return.append([term, value_range, tot_value, position])
-    
-        return jsonify(to_return)
-
-
-
-@app.route("/showsavedpaste/") #completely shows the paste in a new tab
-def showsavedpaste():
-    return showpaste(0)
-
-
-@app.route("/showpreviewpaste/")
-def showpreviewpaste():
-    return showpaste(max_preview_modal)
-
-
-@app.route("/getmoredata/")
-def getmoredata():
-    requested_path = request.args.get('paste', '')
-    paste = Paste.Paste(requested_path)
-    p_content = paste.get_p_content().decode('utf-8', 'ignore')
-    to_return = p_content[max_preview_modal-1:]
-    return to_return
-
 
 if __name__ == "__main__":
     app.run(host='0.0.0.0', port=7000, threaded=True)

var/www/Flasks/Flask_browsepastes.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python2
+# -*-coding:UTF-8 -*
+
+'''
+    Flask functions and routes for the trending modules page
+'''
+import redis
+import json
+import flask
+from flask import Flask, render_template, jsonify, request
+
+import Paste
+
+# ============ VARIABLES ============
+import Flask_config
+
+app = Flask_config.app
+cfg = Flask_config.cfg
+max_preview_char = Flask_config.max_preview_char
+max_preview_modal = Flask_config.max_preview_modal
+r_serv_db = Flask_config.r_serv_db
+# ============ FUNCTIONS ============
+
+def getPastebyType(server, module_name):
+    all_path = []
+    for path in server.smembers('WARNING_'+module_name):
+        all_path.append(path)
+    return all_path
+
+
+def event_stream_getImportantPasteByModule(module_name):
+    index = 0
+    all_pastes_list = getPastebyType(r_serv_db, module_name)
+    for path in all_pastes_list:
+        index += 1
+        paste = Paste.Paste(path)
+        content = paste.get_p_content().decode('utf8', 'ignore')
+        content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
+        curr_date = str(paste._get_p_date())
+        curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
+        data = {}
+        data["module"] = module_name
+        data["index"] = index
+        data["path"] = path
+        data["content"] = content[0:content_range]
+        data["linenum"] = paste.get_lines_info()[0]
+        data["date"] = curr_date
+        data["char_to_display"] = max_preview_modal
+        data["finished"] = True if index == len(all_pastes_list) else False
+        yield 'retry: 100000\ndata: %s\n\n' % json.dumps(data) #retry to avoid reconnection of the browser
+
+# ============ ROUTES ============
+
+@app.route("/browseImportantPaste/", methods=['GET'])
+def browseImportantPaste():
+    module_name = request.args.get('moduleName')
+    return render_template("browse_important_paste.html")
+
+
+@app.route("/importantPasteByModule/", methods=['GET'])
+def importantPasteByModule():
+    module_name = request.args.get('moduleName')
+
+    all_content = []
+    paste_date = []
+    paste_linenum = []
+    all_path = []
+    allPastes = getPastebyType(r_serv_db, module_name)
+
+    for path in allPastes[0:10]:
+        all_path.append(path)
+        paste = Paste.Paste(path)
+        content = paste.get_p_content().decode('utf8', 'ignore')
+        content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
+        all_content.append(content[0:content_range].replace("\"", "\'").replace("\r", " ").replace("\n", " "))
+        curr_date = str(paste._get_p_date())
+        curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
+        paste_date.append(curr_date)
+        paste_linenum.append(paste.get_lines_info()[0])
+
+    if len(allPastes) > 10:
+        finished = False
+    else:
+        finished = True
+
+    return render_template("important_paste_by_module.html",
+            moduleName=module_name, 
+            all_path=all_path, 
+            content=all_content, 
+            paste_date=paste_date, 
+            paste_linenum=paste_linenum, 
+            char_to_display=max_preview_modal, 
+            finished=finished)
+
+@app.route("/_getImportantPasteByModule")
+def getImportantPasteByModule():
+    module_name = request.args.get('moduleName')
+    return flask.Response(event_stream_getImportantPasteByModule(module_name), mimetype="text/event-stream")
+
+

var/www/Flasks/Flask_config.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python2
+# -*-coding:UTF-8 -*
+
+'''
+    Flask global variables shared accross modules
+'''
+import ConfigParser
+import redis
+import os
+
+# FLASK #
+app = None
+
+# CONFIG #
+configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
+if not os.path.exists(configfile):
+    raise Exception('Unable to find the configuration file. \
+                    Did you set environment variables? \
+                    Or activate the virtualenv.')
+
+cfg = ConfigParser.ConfigParser()
+cfg.read(configfile)
+
+
+# REDIS #
+r_serv = redis.StrictRedis(
+    host=cfg.get("Redis_Queues", "host"),
+    port=cfg.getint("Redis_Queues", "port"),
+    db=cfg.getint("Redis_Queues", "db"))
+
+r_serv_log = redis.StrictRedis(
+    host=cfg.get("Redis_Log", "host"),
+    port=cfg.getint("Redis_Log", "port"),
+    db=cfg.getint("Redis_Log", "db"))
+
+r_serv_charts = redis.StrictRedis(
+    host=cfg.get("Redis_Level_DB_Trending", "host"),
+    port=cfg.getint("Redis_Level_DB_Trending", "port"),
+    db=cfg.getint("Redis_Level_DB_Trending", "db"))
+
+r_serv_db = redis.StrictRedis(
+    host=cfg.get("Redis_Level_DB", "host"),
+    port=cfg.getint("Redis_Level_DB", "port"),
+    db=cfg.getint("Redis_Level_DB", "db"))
+
+r_serv_sentiment = redis.StrictRedis(
+        host=cfg.get("Redis_Level_DB_Sentiment", "host"),
+        port=cfg.getint("Redis_Level_DB_Sentiment", "port"),
+        db=cfg.getint("Redis_Level_DB_Sentiment", "db"))
+
+r_serv_term = redis.StrictRedis(
+        host=cfg.get("Redis_Level_DB_TermFreq", "host"),
+        port=cfg.getint("Redis_Level_DB_TermFreq", "port"),
+        db=cfg.getint("Redis_Level_DB_TermFreq", "db"))
+
+r_serv_pasteName = redis.StrictRedis(
+    host=cfg.get("Redis_Paste_Name", "host"),
+    port=cfg.getint("Redis_Paste_Name", "port"),
+    db=cfg.getint("Redis_Paste_Name", "db"))
+
+# VARIABLES #
+max_preview_char = int(cfg.get("Flask", "max_preview_char")) # Maximum number of character to display in the tooltip
+max_preview_modal = int(cfg.get("Flask", "max_preview_modal")) # Maximum number of character to display in the modal
+
+tlsh_to_percent = 1000.0 #Use to display the estimated percentage instead of a raw value

var/www/Flasks/Flask_corpus.py

@@ -0,0 +1,219 @@
+#!/usr/bin/env python2
+# -*-coding:UTF-8 -*
+
+'''
+    Flask functions and routes for the trending modules page
+'''
+import redis
+import datetime
+import calendar
+import flask
+from flask import Flask, render_template, jsonify, request
+
+import Paste
+
+# ============ VARIABLES ============
+import Flask_config
+
+app = Flask_config.app
+cfg = Flask_config.cfg
+r_serv_corpus = Flask_config.r_serv_corpus
+# ============ FUNCTIONS ============
+
+def Corpus_getValueOverRange(word, startDate, num_day):
+    passed_days = 0
+    oneDay = 60*60*24
+    to_return = []
+    curr_to_return = 0
+    for timestamp in range(startDate, startDate - max(num_day)*oneDay, -oneDay):
+        value = r_serv_corpus.hget(timestamp, word)
+        curr_to_return += int(value) if value is not None else 0
+        for i in num_day:
+            if passed_days == i-1:
+                to_return.append(curr_to_return)
+        passed_days += 1
+    return to_return
+
+
+# ============ ROUTES ============
+
+@app.route("/corpus_management/")
+def corpus_management():
+    TrackedCorpusSet_Name = "TrackedSetCorpusSet"
+    TrackedCorpusDate_Name = "TrackedCorpusDate"
+    
+    today = datetime.datetime.now()
+    today = today.replace(hour=0, minute=0, second=0, microsecond=0)
+    today_timestamp = calendar.timegm(today.timetuple())
+
+    track_list = []
+    track_list_values = []
+    track_list_num_of_paste = []
+    for tracked_corpus in r_serv_corpus.smembers(TrackedCorpusSet_Name):
+        track_list.append(tracked_corpus)
+        value_range = Corpus_getValueOverRange(tracked_corpus, today_timestamp, [1, 7, 31])
+
+        corpus_date = r_serv_corpus.hget(TrackedCorpusDate_Name, tracked_corpus)
+
+        set_paste_name = "tracked_" + tracked_corpus
+        track_list_num_of_paste.append(r_serv_corpus.scard(set_paste_name))
+        corpus_date = datetime.datetime.utcfromtimestamp(int(corpus_date)) if corpus_date is not None else "No date recorded"
+        value_range.append(corpus_date)
+        track_list_values.append(value_range)
+
+
+    return render_template("corpus_management.html", black_list=black_list, track_list=track_list, track_list_values=track_list_values,  track_list_num_of_paste=track_list_num_of_paste)
+
+
+@app.route("/corpus_management_query_paste/")
+def corpus_management_query_paste():
+    corpus =  request.args.get('corpus')
+    TrackedCorpusSet_Name = "TrackedSetCorpusSet"
+    paste_info = []
+
+    set_paste_name = "tracked_" + corpus
+    track_list_path = r_serv_corpus.smembers(set_paste_name)
+
+    for path in track_list_path:
+        paste = Paste.Paste(path)
+        p_date = str(paste._get_p_date())
+        p_date = p_date[6:]+'/'+p_date[4:6]+'/'+p_date[0:4]
+        p_source = paste.p_source
+        p_encoding = paste._get_p_encoding()
+        p_size = paste.p_size
+        p_mime = paste.p_mime
+        p_lineinfo = paste.get_lines_info()
+        p_content = paste.get_p_content().decode('utf-8', 'ignore')
+        if p_content != 0:
+            p_content = p_content[0:400]
+        paste_info.append({"path": path, "date": p_date, "source": p_source, "encoding": p_encoding, "size": p_size, "mime": p_mime, "lineinfo": p_lineinfo, "content": p_content})
+
+    return jsonify(paste_info)
+
+
+@app.route("/corpus_management_query/")
+def corpus_management_query():
+    TrackedCorpusDate_Name = "TrackedCorpusDate"
+    corpus =  request.args.get('corpus')
+
+    today = datetime.datetime.now()
+    today = today.replace(hour=0, minute=0, second=0, microsecond=0)
+    today_timestamp = calendar.timegm(today.timetuple())
+    value_range = corpus_getValueOverRange(corpus, today_timestamp, [1, 7, 31])
+
+    corpus_date = r_serv_corpus.hget(TrackedCorpusDate_Name, corpus)
+
+    corpus_date = datetime.datetime.utcfromtimestamp(int(corpus_date)) if corpus_date is not None else "No date recorded"
+    value_range.append(str(corpus_date))
+    return jsonify(value_range)
+
+
+@app.route("/corpus_management_action/", methods=['GET'])
+def corpus_management_action():
+    TrackedCorpusSet_Name = "TrackedSetCorpusSet"
+    TrackedCorpusDate_Name = "TrackedCorpusDate"
+
+    today = datetime.datetime.now()
+    today = today.replace(microsecond=0)
+    today_timestamp = calendar.timegm(today.timetuple())
+
+
+    section = request.args.get('section')
+    action = request.args.get('action')
+    corpus =  request.args.get('corpus')
+    if action is None or corpus is None:
+        return "None"
+    else:
+        if section == "followCorpus":
+            if action == "add":
+                r_serv_corpus.sadd(TrackedCorpusSet_Name, corpus.lower())
+                r_serv_corpus.hset(TrackedCorpusDate_Name, corpus, today_timestamp)
+            else:
+                r_serv_corpus.srem(TrackedCorpusSet_Name, corpus.lower())
+        else:
+            return "None"
+
+        to_return = {}
+        to_return["section"] = section
+        to_return["action"] = action
+        to_return["corpus"] = corpus
+        return jsonify(to_return)
+
+
+
+@app.route("/corpus_plot_tool/")
+def corpus_plot_tool():
+    corpus =  request.args.get('corpus')
+    if corpus is not None:
+        return render_template("corpus_plot_tool.html", corpus=corpus)
+    else:
+        return render_template("corpus_plot_tool.html", corpus="")
+
+
+@app.route("/corpus_plot_tool_data/")
+def corpus_plot_tool_data():
+    oneDay = 60*60*24
+    range_start =  datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_start')))) if request.args.get('range_start') is not None else 0;
+    range_start = range_start.replace(hour=0, minute=0, second=0, microsecond=0)
+    range_start = calendar.timegm(range_start.timetuple())
+    range_end =  datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_end')))) if request.args.get('range_end') is not None else 0;
+    range_end = range_end.replace(hour=0, minute=0, second=0, microsecond=0)
+    range_end = calendar.timegm(range_end.timetuple())
+    corpus =  request.args.get('corpus')
+
+    if corpus is None:
+        return "None"
+    else:
+        value_range = []
+        for timestamp in range(range_start, range_end+oneDay, oneDay):
+            value = r_serv_corpus.hget(timestamp, corpus)
+            curr_value_range = int(value) if value is not None else 0
+            value_range.append([timestamp, curr_value_range])
+        value_range.insert(0,corpus)
+        return jsonify(value_range)
+
+
+@app.route("/corpus_plot_top/")
+def corpus_plot_top():
+    return render_template("corpus_plot_top.html")
+
+
+@app.route("/corpus_plot_top_data/")
+def corpus_plot_top_data():
+    oneDay = 60*60*24
+    today = datetime.datetime.now()
+    today = today.replace(hour=0, minute=0, second=0, microsecond=0)
+    today_timestamp = calendar.timegm(today.timetuple())
+
+    set_day = "TopCorpusFreq_set_day_" + str(today_timestamp)
+    set_week = "TopCorpusFreq_set_week";
+    set_month = "TopCorpusFreq_set_month";
+
+    the_set = request.args.get('set')
+    num_day = int(request.args.get('num_day'))
+    if the_set is None:
+        return "None"
+    else:
+        to_return = []
+        if the_set == "TopCorpusFreq_set_day":
+            the_set += "_" + str(today_timestamp)
+
+        for corpus, tot_value in r_serv_corpus.zrevrangebyscore(the_set, '+inf', '-inf', withscores=True, start=0, num=20):
+            position = {}
+            position['day'] = r_serv_corpus.zrevrank(set_day, corpus)
+            position['day'] = position['day']+1 if position['day'] is not None else "<20"
+            position['week'] = r_serv_corpus.zrevrank(set_week, corpus)
+            position['week'] = position['week']+1 if position['week'] is not None else "<20"
+            position['month'] = r_serv_corpus.zrevrank(set_month, corpus)
+            position['month'] = position['month']+1 if position['month'] is not None else "<20"
+            value_range = []
+            for timestamp in range(today_timestamp, today_timestamp - num_day*oneDay, -oneDay):
+                value = r_serv_corpus.hget(timestamp, corpus)
+                curr_value_range = int(value) if value is not None else 0
+                value_range.append([timestamp, curr_value_range])
+                
+            to_return.append([corpus, value_range, tot_value, position])
+    
+        return jsonify(to_return)
+
+

var/www/Flasks/Flask_dashboard.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python2
+# -*-coding:UTF-8 -*
+
+'''
+    Flask functions and routes for the dashboard page
+'''
+import json
+
+import datetime
+import flask
+from flask import Flask, render_template, jsonify, request
+
+# ============ VARIABLES ============
+import Flask_config
+
+app = Flask_config.app
+cfg = Flask_config.cfg
+r_serv = Flask_config.r_serv
+r_serv_log = Flask_config.r_serv_log
+# ============ FUNCTIONS ============
+
+def event_stream():
+    pubsub = r_serv_log.pubsub()
+    pubsub.psubscribe("Script" + '.*')
+    for msg in pubsub.listen():
+        level = msg['channel'].split('.')[1]
+        if msg['type'] == 'pmessage' and level != "DEBUG":
+            yield 'data: %s\n\n' % json.dumps(msg)
+
+def get_queues(r):
+    # We may want to put the llen in a pipeline to do only one query.
+    newData = []
+    for queue, card in r.hgetall("queues").iteritems():
+        key = "MODULE_" + queue + "_"
+        keySet = "MODULE_TYPE_" + queue
+
+        for moduleNum in r.smembers(keySet):
+    
+            value = r.get(key + str(moduleNum))
+            if value is not None:
+                timestamp, path = value.split(", ")
+                if timestamp is not None:
+                    startTime_readable = datetime.datetime.fromtimestamp(int(timestamp))
+                    processed_time_readable = str((datetime.datetime.now() - startTime_readable)).split('.')[0]
+                    seconds = int((datetime.datetime.now() - startTime_readable).total_seconds())
+                    newData.append( (queue, card, seconds, moduleNum) )
+                else:
+                    newData.append( (queue, cards, 0, moduleNum) )
+
+    return newData
+
+# ============ ROUTES ============
+
+@app.route("/_logs")
+def logs():
+    return flask.Response(event_stream(), mimetype="text/event-stream")
+
+
+@app.route("/_stuff", methods=['GET'])
+def stuff():
+    return jsonify(row1=get_queues(r_serv))
+
+
+@app.route("/")
+def index():
+    default_minute = cfg.get("Flask", "minute_processed_paste")
+    threshold_stucked_module = cfg.getint("Module_ModuleInformation", "threshold_stucked_module")
+    return render_template("index.html", default_minute = default_minute, threshold_stucked_module=threshold_stucked_module)

var/www/Flasks/Flask_search.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python2
+# -*-coding:UTF-8 -*
+
+'''
+    Flask functions and routes for the trending modules page
+'''
+import redis
+import json
+import os
+import flask
+from flask import Flask, render_template, jsonify, request
+
+import Paste
+
+# ============ VARIABLES ============
+import Flask_config
+
+app = Flask_config.app
+cfg = Flask_config.cfg
+r_serv_pasteName = Flask_config.r_serv_pasteName
+max_preview_char = Flask_config.max_preview_char
+max_preview_modal = Flask_config.max_preview_modal
+# ============ FUNCTIONS ============
+
+
+# ============ ROUTES ============
+
+@app.route("/search", methods=['POST'])
+def search():
+    query = request.form['query']
+    q = []
+    q.append(query)
+    r = [] #complete path
+    c = [] #preview of the paste content
+    paste_date = []
+    paste_size = []
+    num_elem_to_get = 50
+
+    # Search filename
+    for path in r_serv_pasteName.smembers(q[0]):
+        r.append(path)
+        paste = Paste.Paste(path)
+        content = paste.get_p_content().decode('utf8', 'ignore')
+        content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
+        c.append(content[0:content_range])
+        curr_date = str(paste._get_p_date())
+        curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
+        paste_date.append(curr_date)
+        paste_size.append(paste._get_p_size())
+
+    # Search full line
+    from whoosh import index
+    from whoosh.fields import Schema, TEXT, ID
+    schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT)
+
+    indexpath = os.path.join(os.environ['AIL_HOME'], cfg.get("Indexer", "path"))
+    ix = index.open_dir(indexpath)
+    from whoosh.qparser import QueryParser
+    with ix.searcher() as searcher:
+        query = QueryParser("content", ix.schema).parse(" ".join(q))
+        results = searcher.search_page(query, 1, pagelen=num_elem_to_get)
+        for x in results:
+            r.append(x.items()[0][1])
+            paste = Paste.Paste(x.items()[0][1])
+            content = paste.get_p_content().decode('utf8', 'ignore')
+            content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
+            c.append(content[0:content_range])
+            curr_date = str(paste._get_p_date())
+            curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
+            paste_date.append(curr_date)
+            paste_size.append(paste._get_p_size())
+        results = searcher.search(query)
+        num_res = len(results)
+
+    return render_template("search.html", r=r, c=c, query=request.form['query'], paste_date=paste_date, paste_size=paste_size, char_to_display=max_preview_modal, num_res=num_res)
+
+
+@app.route("/get_more_search_result", methods=['POST'])
+def get_more_search_result():
+    query = request.form['query']
+    q = []
+    q.append(query)
+    page_offset = int(request.form['page_offset'])
+    num_elem_to_get = 50
+
+    path_array = []
+    preview_array = []
+    date_array = []
+    size_array = []
+
+    from whoosh import index
+    from whoosh.fields import Schema, TEXT, ID
+    schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT)
+
+    indexpath = os.path.join(os.environ['AIL_HOME'], cfg.get("Indexer", "path"))
+    ix = index.open_dir(indexpath)
+    from whoosh.qparser import QueryParser
+    with ix.searcher() as searcher:
+        query = QueryParser("content", ix.schema).parse(" ".join(q))
+        results = searcher.search_page(query, page_offset, num_elem_to_get)   
+        for x in results:
+            path_array.append(x.items()[0][1])
+            paste = Paste.Paste(x.items()[0][1])
+            content = paste.get_p_content().decode('utf8', 'ignore')
+            content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
+            preview_array.append(content[0:content_range])
+            curr_date = str(paste._get_p_date())
+            curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
+            date_array.append(curr_date)
+            size_array.append(paste._get_p_size())
+        to_return = {}
+        to_return["path_array"] = path_array
+        to_return["preview_array"] = preview_array
+        to_return["date_array"] = date_array
+        to_return["size_array"] = size_array
+        print "len(path_array)="+str(len(path_array))
+        if len(path_array) < num_elem_to_get: #pagelength
+            to_return["moreData"] = False
+        else:
+            to_return["moreData"] = True
+
+    return jsonify(to_return)
+
+

var/www/Flasks/Flask_sentiment.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python2
+# -*-coding:UTF-8 -*
+
+'''
+    Flask functions and routes for the trending modules page
+'''
+import redis
+import datetime
+import calendar
+from Date import Date
+import flask
+from flask import Flask, render_template, jsonify, request
+
+import Paste
+
+# ============ VARIABLES ============
+import Flask_config
+
+app = Flask_config.app
+cfg = Flask_config.cfg
+r_serv_charts = Flask_config.r_serv_charts
+r_serv_sentiment = Flask_config.r_serv_sentiment
+# ============ FUNCTIONS ============
+
+def get_date_range(num_day):
+    curr_date = datetime.date.today()
+    date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2))
+    date_list = []
+
+    for i in range(0, num_day+1):
+        date_list.append(date.substract_day(i))
+    return date_list
+
+
+# ============ ROUTES ============
+
+@app.route("/sentiment_analysis_trending/")
+def sentiment_analysis_trending():
+    return render_template("sentiment_analysis_trending.html")
+
+
+@app.route("/sentiment_analysis_getplotdata/", methods=['GET'])
+def sentiment_analysis_getplotdata():
+    # Get the top providers based on number of pastes
+    oneHour = 60*60
+    sevenDays = oneHour*24*7
+    dateStart = datetime.datetime.now()
+    dateStart = dateStart.replace(minute=0, second=0, microsecond=0)
+    dateStart_timestamp = calendar.timegm(dateStart.timetuple())
+
+    getAllProviders = request.args.get('getProviders')
+    provider = request.args.get('provider')
+    allProvider = request.args.get('all')
+    if getAllProviders == 'True':
+        if allProvider == "True":
+            range_providers = r_serv_charts.smembers('all_provider_set')
+            return jsonify(list(range_providers))
+        else:
+            range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(0)[0], '+inf', '-inf', start=0, num=8)
+            # if empty, get yesterday top providers
+            range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(1)[1], '+inf', '-inf', start=0, num=8) if range_providers == [] else range_providers
+            # if still empty, takes from all providers
+            if range_providers == []:
+                print 'today provider empty'
+                range_providers = r_serv_charts.smembers('all_provider_set')
+            return jsonify(list(range_providers))
+
+    elif provider is not None:
+        to_return = {}
+
+        cur_provider_name = provider + '_'
+        list_date = {}
+        for cur_timestamp in range(int(dateStart_timestamp), int(dateStart_timestamp)-sevenDays-oneHour, -oneHour):
+            cur_set_name = cur_provider_name + str(cur_timestamp)
+
+            list_value = []
+            for cur_id in r_serv_sentiment.smembers(cur_set_name):
+                cur_value = r_serv_sentiment.get(cur_id)
+                list_value.append(cur_value)
+            list_date[cur_timestamp] = list_value
+        to_return[provider] = list_date
+
+        return jsonify(to_return)
+    return "Bad request"
+
+
+
+@app.route("/sentiment_analysis_plot_tool/")
+def sentiment_analysis_plot_tool():
+    return render_template("sentiment_analysis_plot_tool.html")
+
+
+
+@app.route("/sentiment_analysis_plot_tool_getdata/", methods=['GET'])
+def sentiment_analysis_plot_tool_getdata():
+    getProviders = request.args.get('getProviders')
+
+    if getProviders == 'True':
+        providers = []
+        for cur_provider in r_serv_charts.smembers('all_provider_set'):
+            providers.append(cur_provider)
+        return jsonify(providers)
+
+    else:
+        query = request.args.get('query')
+        query = query.split(',')
+        Qdate = request.args.get('Qdate')
+
+        date1 = (Qdate.split('-')[0]).split('.')
+        date1 = datetime.date(int(date1[2]), int(date1[1]), int(date1[0]))
+
+        date2 = (Qdate.split('-')[1]).split('.')
+        date2 = datetime.date(int(date2[2]), int(date2[1]), int(date2[0]))
+
+        timestamp1 = calendar.timegm(date1.timetuple())
+        timestamp2 = calendar.timegm(date2.timetuple())
+
+        oneHour = 60*60
+        oneDay = oneHour*24
+
+        to_return = {}
+        for cur_provider in query:
+            list_date = {}
+            cur_provider_name = cur_provider + '_'
+            for cur_timestamp in range(int(timestamp1), int(timestamp2)+oneDay, oneHour):
+                cur_set_name = cur_provider_name + str(cur_timestamp)
+
+                list_value = []
+                for cur_id in r_serv_sentiment.smembers(cur_set_name):
+                    cur_value = r_serv_sentiment.get(cur_id)
+                    list_value.append(cur_value)
+                list_date[cur_timestamp] = list_value
+            to_return[cur_provider] = list_date
+
+        return jsonify(to_return)
+
+

var/www/Flasks/Flask_showpaste.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python2
+# -*-coding:UTF-8 -*
+
+'''
+    Flask functions and routes for the trending modules page
+'''
+import redis
+import json
+import flask
+from flask import Flask, render_template, jsonify, request
+
+import Paste
+
+# ============ VARIABLES ============
+import Flask_config
+
+app = Flask_config.app
+cfg = Flask_config.cfg
+r_serv_pasteName = Flask_config.r_serv_pasteName
+max_preview_char = Flask_config.max_preview_char
+max_preview_modal = Flask_config.max_preview_modal
+tlsh_to_percent = Flask_config.tlsh_to_percent
+# ============ FUNCTIONS ============
+
+def showpaste(content_range):
+    requested_path = request.args.get('paste', '')
+    paste = Paste.Paste(requested_path)
+    p_date = str(paste._get_p_date())
+    p_date = p_date[6:]+'/'+p_date[4:6]+'/'+p_date[0:4]
+    p_source = paste.p_source
+    p_encoding = paste._get_p_encoding()
+    p_language = paste._get_p_language()
+    p_size = paste.p_size
+    p_mime = paste.p_mime
+    p_lineinfo = paste.get_lines_info()
+    p_content = paste.get_p_content().decode('utf-8', 'ignore')
+    p_duplicate_full_list = json.loads(paste._get_p_duplicate())
+    p_duplicate_list = []
+    p_simil_list = []
+    p_date_list = []
+    p_hashtype_list = []
+
+
+    for dup_list in p_duplicate_full_list:
+        if dup_list[0] == "tlsh":
+            dup_list[2] = int(((tlsh_to_percent - float(dup_list[2])) / tlsh_to_percent)*100)
+        else:
+            dup_list[2] = int(dup_list[2])
+
+    p_duplicate_full_list.sort(lambda x,y: cmp(x[2], y[2]), reverse=True)
+
+    # Combine multiple duplicate paste name and format for display
+    new_dup_list = []
+    dup_list_removed = []
+    for dup_list_index in range(0, len(p_duplicate_full_list)):
+        if dup_list_index in dup_list_removed:
+            continue
+        indices = [i for i, x in enumerate(p_duplicate_full_list) if x[1] == p_duplicate_full_list[dup_list_index][1]]
+        hash_types = []
+        comp_vals = []
+        for i in indices:
+            hash_types.append(p_duplicate_full_list[i][0].encode('utf8'))
+            comp_vals.append(p_duplicate_full_list[i][2])
+            dup_list_removed.append(i)
+
+        hash_types = str(hash_types).replace("[","").replace("]","") if len(hash_types)==1 else str(hash_types)
+        comp_vals = str(comp_vals).replace("[","").replace("]","") if len(comp_vals)==1 else str(comp_vals)
+        if len(p_duplicate_full_list[dup_list_index]) > 3:
+            try:
+                date_paste = str(int(p_duplicate_full_list[dup_list_index][3]))
+                date_paste = date_paste[0:4]+"-"+date_paste[4:6]+"-"+date_paste[6:8]
+            except ValueError:
+                date_paste = str(p_duplicate_full_list[dup_list_index][3])
+        else:
+            date_paste = "No date available"
+        new_dup_list.append([hash_types.replace("'", ""), p_duplicate_full_list[dup_list_index][1], comp_vals, date_paste])
+
+    # Create the list to pass to the webpage
+    for dup_list in new_dup_list:
+        hash_type, path, simil_percent, date_paste = dup_list
+        p_duplicate_list.append(path)
+        p_simil_list.append(simil_percent)
+        p_hashtype_list.append(hash_type)
+        p_date_list.append(date_paste)
+
+    if content_range != 0:
+       p_content = p_content[0:content_range]
+
+
+    return render_template("show_saved_paste.html", date=p_date, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=len(p_content), duplicate_list = p_duplicate_list, simil_list = p_simil_list, hashtype_list = p_hashtype_list, date_list=p_date_list)
+
+
+
+# ============ ROUTES ============
+
+@app.route("/showsavedpaste/") #completely shows the paste in a new tab
+def showsavedpaste():
+    return showpaste(0)
+
+
+@app.route("/showpreviewpaste/")
+def showpreviewpaste():
+    num = request.args.get('num', '')
+    return "|num|"+num+"|num|"+showpaste(max_preview_modal)
+
+
+@app.route("/getmoredata/")
+def getmoredata():
+    requested_path = request.args.get('paste', '')
+    paste = Paste.Paste(requested_path)
+    p_content = paste.get_p_content().decode('utf-8', 'ignore')
+    to_return = p_content[max_preview_modal-1:]
+    return to_return
+

var/www/Flasks/Flask_terms.py

@@ -0,0 +1,240 @@
+#!/usr/bin/env python2
+# -*-coding:UTF-8 -*
+
+'''
+    Flask functions and routes for the trending modules page
+'''
+import redis
+import datetime
+import calendar
+import flask
+from flask import Flask, render_template, jsonify, request
+
+import Paste
+
+# ============ VARIABLES ============
+import Flask_config
+
+app = Flask_config.app
+cfg = Flask_config.cfg
+r_serv_term = Flask_config.r_serv_term
+# ============ FUNCTIONS ============
+
+def Term_getValueOverRange(word, startDate, num_day):
+    passed_days = 0
+    oneDay = 60*60*24
+    to_return = []
+    curr_to_return = 0
+    for timestamp in range(startDate, startDate - max(num_day)*oneDay, -oneDay):
+        value = r_serv_term.hget(timestamp, word)
+        curr_to_return += int(value) if value is not None else 0
+        for i in num_day:
+            if passed_days == i-1:
+                to_return.append(curr_to_return)
+        passed_days += 1
+    return to_return
+
+
+# ============ ROUTES ============
+
+@app.route("/terms_management/")
+def terms_management():
+    TrackedTermsSet_Name = "TrackedSetTermSet"
+    BlackListTermsSet_Name = "BlackListSetTermSet"
+    TrackedTermsDate_Name = "TrackedTermDate"
+    BlackListTermsDate_Name = "BlackListTermDate"
+    
+    today = datetime.datetime.now()
+    today = today.replace(hour=0, minute=0, second=0, microsecond=0)
+    today_timestamp = calendar.timegm(today.timetuple())
+
+    track_list = []
+    track_list_values = []
+    track_list_num_of_paste = []
+    for tracked_term in r_serv_term.smembers(TrackedTermsSet_Name):
+        track_list.append(tracked_term)
+        value_range = Term_getValueOverRange(tracked_term, today_timestamp, [1, 7, 31])
+
+        term_date = r_serv_term.hget(TrackedTermsDate_Name, tracked_term)
+
+        set_paste_name = "tracked_" + tracked_term
+        track_list_num_of_paste.append(r_serv_term.scard(set_paste_name))
+        term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded"
+        value_range.append(term_date)
+        track_list_values.append(value_range)
+
+
+    black_list = []
+    for blacked_term in r_serv_term.smembers(BlackListTermsSet_Name):
+        term_date = r_serv_term.hget(BlackListTermsDate_Name, blacked_term)
+        term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded"
+        black_list.append([blacked_term, term_date])
+
+    return render_template("terms_management.html", black_list=black_list, track_list=track_list, track_list_values=track_list_values,  track_list_num_of_paste=track_list_num_of_paste)
+
+
+@app.route("/terms_management_query_paste/")
+def terms_management_query_paste():
+    term =  request.args.get('term')
+    TrackedTermsSet_Name = "TrackedSetTermSet"
+    paste_info = []
+
+    set_paste_name = "tracked_" + term
+    track_list_path = r_serv_term.smembers(set_paste_name)
+
+    for path in track_list_path:
+        paste = Paste.Paste(path)
+        p_date = str(paste._get_p_date())
+        p_date = p_date[6:]+'/'+p_date[4:6]+'/'+p_date[0:4]
+        p_source = paste.p_source
+        p_encoding = paste._get_p_encoding()
+        p_size = paste.p_size
+        p_mime = paste.p_mime
+        p_lineinfo = paste.get_lines_info()
+        p_content = paste.get_p_content().decode('utf-8', 'ignore')
+        if p_content != 0:
+            p_content = p_content[0:400]
+        paste_info.append({"path": path, "date": p_date, "source": p_source, "encoding": p_encoding, "size": p_size, "mime": p_mime, "lineinfo": p_lineinfo, "content": p_content})
+
+    return jsonify(paste_info)
+
+
+@app.route("/terms_management_query/")
+def terms_management_query():
+    TrackedTermsDate_Name = "TrackedTermDate"
+    BlackListTermsDate_Name = "BlackListTermDate"
+    term =  request.args.get('term')
+    section = request.args.get('section')
+
+    today = datetime.datetime.now()
+    today = today.replace(hour=0, minute=0, second=0, microsecond=0)
+    today_timestamp = calendar.timegm(today.timetuple())
+    value_range = Term_getValueOverRange(term, today_timestamp, [1, 7, 31])
+
+    if section == "followTerm":
+        term_date = r_serv_term.hget(TrackedTermsDate_Name, term)
+    elif section == "blacklistTerm":
+        term_date = r_serv_term.hget(BlackListTermsDate_Name, term)
+
+    term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded"
+    value_range.append(str(term_date))
+    return jsonify(value_range)
+
+
+@app.route("/terms_management_action/", methods=['GET'])
+def terms_management_action():
+    TrackedTermsSet_Name = "TrackedSetTermSet"
+    TrackedTermsDate_Name = "TrackedTermDate"
+    BlackListTermsDate_Name = "BlackListTermDate"
+    BlackListTermsSet_Name = "BlackListSetTermSet"
+
+    today = datetime.datetime.now()
+    today = today.replace(microsecond=0)
+    today_timestamp = calendar.timegm(today.timetuple())
+
+
+    section = request.args.get('section')
+    action = request.args.get('action')
+    term =  request.args.get('term')
+    if action is None or term is None:
+        return "None"
+    else:
+        if section == "followTerm":
+            if action == "add":
+                r_serv_term.sadd(TrackedTermsSet_Name, term.lower())
+                r_serv_term.hset(TrackedTermsDate_Name, term, today_timestamp)
+            else:
+                r_serv_term.srem(TrackedTermsSet_Name, term.lower())
+        elif section == "blacklistTerm":
+            if action == "add":
+                r_serv_term.sadd(BlackListTermsSet_Name, term.lower())
+                r_serv_term.hset(BlackListTermsDate_Name, term, today_timestamp)
+            else:
+                r_serv_term.srem(BlackListTermsSet_Name, term.lower())
+        else:
+            return "None"
+
+        to_return = {}
+        to_return["section"] = section
+        to_return["action"] = action
+        to_return["term"] = term
+        return jsonify(to_return)
+
+
+
+@app.route("/terms_plot_tool/")
+def terms_plot_tool():
+    term =  request.args.get('term')
+    if term is not None:
+        return render_template("terms_plot_tool.html", term=term)
+    else:
+        return render_template("terms_plot_tool.html", term="")
+
+
+@app.route("/terms_plot_tool_data/")
+def terms_plot_tool_data():
+    oneDay = 60*60*24
+    range_start =  datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_start')))) if request.args.get('range_start') is not None else 0;
+    range_start = range_start.replace(hour=0, minute=0, second=0, microsecond=0)
+    range_start = calendar.timegm(range_start.timetuple())
+    range_end =  datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_end')))) if request.args.get('range_end') is not None else 0;
+    range_end = range_end.replace(hour=0, minute=0, second=0, microsecond=0)
+    range_end = calendar.timegm(range_end.timetuple())
+    term =  request.args.get('term')
+
+    if term is None:
+        return "None"
+    else:
+        value_range = []
+        for timestamp in range(range_start, range_end+oneDay, oneDay):
+            value = r_serv_term.hget(timestamp, term)
+            curr_value_range = int(value) if value is not None else 0
+            value_range.append([timestamp, curr_value_range])
+        value_range.insert(0,term)
+        return jsonify(value_range)
+
+
+@app.route("/terms_plot_top/")
+def terms_plot_top():
+    return render_template("terms_plot_top.html")
+
+
+@app.route("/terms_plot_top_data/")
+def terms_plot_top_data():
+    oneDay = 60*60*24
+    today = datetime.datetime.now()
+    today = today.replace(hour=0, minute=0, second=0, microsecond=0)
+    today_timestamp = calendar.timegm(today.timetuple())
+
+    set_day = "TopTermFreq_set_day_" + str(today_timestamp)
+    set_week = "TopTermFreq_set_week";
+    set_month = "TopTermFreq_set_month";
+
+    the_set = request.args.get('set')
+    num_day = int(request.args.get('num_day'))
+    if the_set is None:
+        return "None"
+    else:
+        to_return = []
+        if the_set == "TopTermFreq_set_day":
+            the_set += "_" + str(today_timestamp)
+
+        for term, tot_value in r_serv_term.zrevrangebyscore(the_set, '+inf', '-inf', withscores=True, start=0, num=20):
+            position = {}
+            position['day'] = r_serv_term.zrevrank(set_day, term)
+            position['day'] = position['day']+1 if position['day'] is not None else "<20"
+            position['week'] = r_serv_term.zrevrank(set_week, term)
+            position['week'] = position['week']+1 if position['week'] is not None else "<20"
+            position['month'] = r_serv_term.zrevrank(set_month, term)
+            position['month'] = position['month']+1 if position['month'] is not None else "<20"
+            value_range = []
+            for timestamp in range(today_timestamp, today_timestamp - num_day*oneDay, -oneDay):
+                value = r_serv_term.hget(timestamp, term)
+                curr_value_range = int(value) if value is not None else 0
+                value_range.append([timestamp, curr_value_range])
+                
+            to_return.append([term, value_range, tot_value, position])
+    
+        return jsonify(to_return)
+
+

var/www/Flasks/Flask_trendingcharts.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python2
+# -*-coding:UTF-8 -*
+
+'''
+    Flask functions and routes for the trending charts page
+'''
+import redis
+import datetime
+from Date import Date
+import flask
+from flask import Flask, render_template, jsonify, request
+
+# ============ VARIABLES ============
+import Flask_config
+
+app = Flask_config.app
+cfg = Flask_config.cfg
+r_serv_charts = Flask_config.r_serv_charts
+# ============ FUNCTIONS ============
+
+def get_date_range(num_day):
+    curr_date = datetime.date.today()
+    date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2))
+    date_list = []
+
+    for i in range(0, num_day+1):
+        date_list.append(date.substract_day(i))
+    return date_list
+
+
+# ============ ROUTES ============
+
+@app.route("/_progressionCharts", methods=['GET'])
+def progressionCharts():
+    attribute_name = request.args.get('attributeName')
+    trending_name = request.args.get('trendingName')
+    bar_requested = True if request.args.get('bar') == "true" else False
+
+    if (bar_requested):
+        num_day = int(request.args.get('days'))
+        bar_values = []
+
+        date_range = get_date_range(num_day)
+        # Retreive all data from the last num_day
+        for date in date_range:
+            curr_value = r_serv_charts.hget(attribute_name, date)
+            bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], int(curr_value if curr_value is not None else 0)])
+        bar_values.insert(0, attribute_name)
+        return jsonify(bar_values)
+
+    else:
+        redis_progression_name = "z_top_progression_" + trending_name
+        keyw_value = r_serv_charts.zrevrangebyscore(redis_progression_name, '+inf', '-inf', withscores=True, start=0, num=10)
+        return jsonify(keyw_value)
+
+@app.route("/wordstrending/")
+def wordstrending():
+    default_display = cfg.get("Flask", "default_display")
+    return render_template("Wordstrending.html", default_display = default_display)
+
+
+@app.route("/protocolstrending/")
+def protocolstrending():
+    default_display = cfg.get("Flask", "default_display")
+    return render_template("Protocolstrending.html", default_display = default_display)
+
+
+@app.route("/trending/")
+def trending():
+    default_display = cfg.get("Flask", "default_display")
+    return render_template("Trending.html", default_display = default_display)
+
+

var/www/Flasks/Flask_trendingmodules.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python2
+# -*-coding:UTF-8 -*
+
+'''
+    Flask functions and routes for the trending modules page
+'''
+import redis
+import datetime
+from Date import Date
+import flask
+from flask import Flask, render_template, jsonify, request
+
+# ============ VARIABLES ============
+import Flask_config
+
+app = Flask_config.app
+cfg = Flask_config.cfg
+r_serv_charts = Flask_config.r_serv_charts
+# ============ FUNCTIONS ============
+
+# Iterate over elements in the module provided and return the today data or the last data
+# return format: [('passed_days', num_of_passed_days), ('elem_name1', elem_value1), ('elem_name2', elem_value2)]]
+def get_top_relevant_data(server, module_name):
+    days = 0
+    for date in get_date_range(15):
+        redis_progression_name_set = 'top_'+ module_name +'_set_' + date
+        member_set = server.zrevrangebyscore(redis_progression_name_set, '+inf', '-inf', withscores=True)
+        if len(member_set) == 0: #No data for this date
+            days += 1
+        else:
+            member_set.insert(0, ("passed_days", days))
+            return member_set
+
+
+def get_date_range(num_day):
+    curr_date = datetime.date.today()
+    date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2))
+    date_list = []
+
+    for i in range(0, num_day+1):
+        date_list.append(date.substract_day(i))
+    return date_list
+
+# ============ ROUTES ============
+
+@app.route("/_moduleCharts", methods=['GET'])
+def modulesCharts():
+    keyword_name = request.args.get('keywordName')
+    module_name = request.args.get('moduleName')
+    bar_requested = True if request.args.get('bar') == "true" else False
+
+    if (bar_requested):
+        num_day = int(request.args.get('days'))
+        bar_values = []
+
+        date_range = get_date_range(num_day)
+        # Retreive all data from the last num_day
+        for date in date_range:
+            curr_value = r_serv_charts.hget(date, module_name+'-'+keyword_name)
+            bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], int(curr_value if curr_value is not None else 0)])
+        bar_values.insert(0, keyword_name)
+        return jsonify(bar_values)
+
+    else:
+        member_set = get_top_relevant_data(r_serv_charts, module_name)
+        member_set = member_set if member_set is not None else []
+        if len(member_set) == 0:
+            member_set.append(("No relevant data", int(100)))
+        return jsonify(member_set)
+
+
+@app.route("/_providersChart", methods=['GET'])
+def providersChart():
+    keyword_name = request.args.get('keywordName')
+    module_name = request.args.get('moduleName')
+    bar_requested = True if request.args.get('bar') == "true" else False
+
+    if (bar_requested):
+        num_day = int(request.args.get('days'))
+        bar_values = []
+
+        date_range = get_date_range(num_day)
+        # Retreive all data from the last num_day
+        for date in date_range:
+            curr_value_size = r_serv_charts.hget(keyword_name+'_'+'size', date)
+            curr_value_num = r_serv_charts.hget(keyword_name+'_'+'num', date)
+            curr_value_size_avg = r_serv_charts.hget(keyword_name+'_'+'avg', date)
+            if module_name == "size":
+                curr_value = float(curr_value_size_avg if curr_value_size_avg is not None else 0)
+            else:
+                curr_value = float(curr_value_num if curr_value_num is not None else 0.0)
+
+            bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], curr_value])
+        bar_values.insert(0, keyword_name)
+        return jsonify(bar_values)
+
+    else:
+        #redis_provider_name_set = 'top_size_set' if module_name == "size" else 'providers_set'
+        redis_provider_name_set = 'top_avg_size_set_' if module_name == "size" else 'providers_set_'
+        redis_provider_name_set = redis_provider_name_set + get_date_range(0)[0]
+        
+        member_set = r_serv_charts.zrevrangebyscore(redis_provider_name_set, '+inf', '-inf', withscores=True, start=0, num=8)
+        # Member set is a list of (value, score) pairs
+        if len(member_set) == 0:
+            member_set.append(("No relevant data", float(100)))
+        return jsonify(member_set)
+
+
+@app.route("/moduletrending/")
+def moduletrending():
+    return render_template("Moduletrending.html")
+
+

var/www/static/js/indexjavascript.js

@@ -1,13 +1,25 @@
-var time_since_last_pastes_num;
+var time_since_last_pastes_num = {};
+var data_for_processed_paste = {  };
+var list_feeder = [];
+window.paste_num_tabvar_all = {};
 
-//If we do not received info from global, set pastes_num to 0
+//If we do not received info from mixer, set pastes_num to 0
 function checkIfReceivedData(){
-    if ((new Date().getTime() - time_since_last_pastes_num) > 45*1000)
+    for (i in list_feeder) {
-        window.paste_num_tabvar = 0;
+        if(list_feeder[i] == "global"){
-    setTimeout(checkIfReceivedData, 45*1000);
+            if ((new Date().getTime() - time_since_last_pastes_num[list_feeder[i]]) > 35*1000){
+                window.paste_num_tabvar_all[list_feeder[i]] = 0;
+            }
+        } else {
+            if ((new Date().getTime() - time_since_last_pastes_num["Proc"+list_feeder[i]]) > 35*1000){
+                window.paste_num_tabvar_all["Proc"+list_feeder[i]] = 0;
+                window.paste_num_tabvar_all["Dup"+list_feeder[i]] = 0;
+            }
+        }
+    }
+    setTimeout(checkIfReceivedData, 35*1000);
 }
 
-setTimeout(checkIfReceivedData, 45*1000);
 
 function initfunc( csvay, scroot) {
   window.csv = csvay;
@@ -24,54 +36,88 @@ function update_values() {
 
 
 // Plot and update the number of processed pastes
-$(function() {
+// BEGIN PROCESSED PASTES
-    var data = [];
     var default_minute = (typeof window.default_minute !== "undefined") ? parseInt(window.default_minute) : 10;
-    var totalPoints = 60*parseInt(default_minute); //60s*minute
+    var totalPoints = 2*parseInt(default_minute); //60s*minute
-    var curr_max = 0;
+    var curr_max = {"global": 0};
 
-    function getData() {
+    function fetch_data(dataset, curr_data, feeder_name) {
-        if (data.length > 0){
+        if (curr_data.length > 0){
-             var data_old = data[0];
+             var data_old = curr_data[0];
-             data = data.slice(1);
+             curr_data = curr_data.slice(1);
-             curr_max = curr_max == data_old ? Math.max.apply(null, data) : curr_max;
+             curr_max[dataset] = curr_max[dataset] == data_old ? Math.max.apply(null, curr_data) : curr_max[dataset];
         }
         
-        while (data.length < totalPoints) {
+        while (curr_data.length < totalPoints) {
-            var y = (typeof window.paste_num_tabvar !== "undefined") ? parseInt(window.paste_num_tabvar) : 0;
+            var y = (typeof window.paste_num_tabvar_all[dataset] !== "undefined") ? parseInt(window.paste_num_tabvar_all[dataset]) : 0;
-            curr_max = y > curr_max ? y : curr_max;
+            curr_max[dataset] = y > curr_max[dataset] ? y : curr_max[dataset];
-            data.push(y);
+            curr_data.push(y);
         }
         // Zip the generated y values with the x values
         var res = [];
-        for (var i = 0; i < data.length; ++i) {
+        for (var i = 0; i < curr_data.length; ++i) {
-            res.push([i, data[i]])
+            res.push([i, curr_data[i]])
         } 
-        return res;
+        data_for_processed_paste[dataset] = curr_data;
+        return { label: feeder_name, data: res };
     }
 
-    var updateInterval = 1000;
+    function getData(dataset_group, graph_type) {
-    var options = {
+        var curr_data;
-        series: { shadowSize: 1 },
+ 
-        lines: { fill: true, fillColor: { colors: [ { opacity: 1 }, { opacity: 0.1 } ] }},
+        var all_res = [];
+        if (dataset_group == "global") {
+            if (data_for_processed_paste["global"] ===  undefined) { // create feeder dataset if not exists yet
+                data_for_processed_paste["global"] = [];
+            }
+            curr_data = data_for_processed_paste["global"];
+            all_res.push(fetch_data("global", curr_data, "global"));
+        } else {   
+
+            for(d_i in list_feeder) {
+                if(list_feeder[d_i] == "global") {
+                    continue;
+                }
+
+                dataset = graph_type+list_feeder[d_i];
+                if (data_for_processed_paste[dataset] ===  undefined) { // create feeder dataset if not exists yet
+                    data_for_processed_paste[dataset] = [];
+                }
+                curr_data = data_for_processed_paste[dataset];
+                all_res.push(fetch_data(dataset, curr_data, list_feeder[d_i]));
+            }
+
+        }
+        return all_res;
+    }
+
+    var updateInterval = 10000;
+    var options_processed_pastes = {
+        series: {   shadowSize: 0 ,
+                    lines: { fill: true, fillColor: { colors: [ { opacity: 1 }, { opacity: 0.1 } ] }}
+                },
         yaxis: { min: 0, max: 40 },
-        colors: ["#a971ff"],
+        xaxis: { ticks: [[0, 0], [2, 1], [4, 2], [6, 3], [8, 4], [10, 5], [12, 6], [14, 7], [16, 8], [18, 9], [20, 10]] },
         grid: {
             tickColor: "#dddddd",
             borderWidth: 0 
         },
+        legend: {
+            show: true,
+            position: "nw",
+        }
     };
-    var plot = $.plot("#realtimechart", [ getData() ], options);
 
-    function update() {
+    function update_processed_pastes(graph, dataset, graph_type) {
-        plot.setData([getData()]);
+        graph.setData(getData(dataset, graph_type));
-        plot.getOptions().yaxes[0].max = curr_max;
+        graph.getOptions().yaxes[0].max = curr_max[dataset];
-        plot.setupGrid();
+        graph.setupGrid();
-        plot.draw();
+        graph.draw();
-        setTimeout(update, updateInterval);
+        setTimeout(function(){ update_processed_pastes(graph, dataset, graph_type); }, updateInterval);
     }
-    update();
+
-});
+
+// END PROCESSED PASTES    
 
 function initfunc( csvay, scroot) {
   window.csv = csvay;
@@ -109,15 +155,49 @@ function create_log_table(obj_json) {
     var pdate = document.createElement('TD')
     var nam = document.createElement('TD')
     var msage = document.createElement('TD')
+    var inspect = document.createElement('TD')
 
     var chansplit = obj_json.channel.split('.');
     var parsedmess = obj_json.data.split(';');
 
   
-    if (parsedmess[0] == "Global"){
+    if (parsedmess[0] == "Mixer"){
-        var paste_processed = parsedmess[4].split(" ")[2];
+        var feeder = parsedmess[4].split(" ")[1];
-        window.paste_num_tabvar = paste_processed;
+        var paste_processed = parsedmess[4].split(" ")[3];
-        time_since_last_pastes_num = new Date().getTime();
+        var msg_type = parsedmess[4].split(" ")[2]; 
+
+        if (feeder == "All_feeders"){
+            if(list_feeder.indexOf("global") == -1) {
+                list_feeder.push("global");
+
+                options_processed_pastes.legend.show = false;
+                var total_proc = $.plot("#global", [ getData("global", null) ], options_processed_pastes);
+                options_processed_pastes.legend.show = true;
+                options_processed_pastes.series.lines = { show: true };
+                data_for_processed_paste["global"] = Array(totalPoints+1).join(0).split('');
+
+                var feederProc = $.plot("#Proc_feeder", [ getData(feeder, "Proc") ], options_processed_pastes);
+                var feederDup = $.plot("#Dup_feeder", [ getData(feeder, "Dup") ], options_processed_pastes);
+
+                update_processed_pastes(feederProc, "feeder", "Proc");
+                update_processed_pastes(feederDup, "feeder", "Dup");
+                update_processed_pastes(total_proc, "global");
+                setTimeout(checkIfReceivedData, 45*1000);
+            }
+           window.paste_num_tabvar_all["global"] = paste_processed;
+           time_since_last_pastes_num["global"] = new Date().getTime();
+        } else {
+
+            if (list_feeder.indexOf(feeder) == -1) {
+                list_feeder.push(feeder);
+                data_for_processed_paste["Proc"+feeder] = Array(totalPoints+1).join(0).split('');
+                data_for_processed_paste["Dup"+feeder] = Array(totalPoints+1).join(0).split('');
+            }
+
+            var feederName = msg_type == "Duplicated" ? "Dup"+feeder : "Proc"+feeder;
+            window.paste_num_tabvar_all[feederName] = paste_processed;
+            time_since_last_pastes_num[feederName] = new Date().getTime();
+        }
         return;
     }
 
@@ -139,7 +219,7 @@ function create_log_table(obj_json) {
         source_url = "http://"+parsedmess[1]+"/"+parsedmess[3].split(".")[0];
     }
     source_link.setAttribute("HREF",source_url);
-    source_link.setAttribute("TARGET", "_blank")
+    source_link.setAttribute("TARGET", "_blank");
     source_link.appendChild(document.createTextNode(parsedmess[1]));
 
     src.appendChild(source_link);
@@ -169,6 +249,18 @@ function create_log_table(obj_json) {
 
     msage.appendChild(document.createTextNode(message.join(" ")));
 
+    var paste_path = parsedmess[5];
+    var url_to_saved_paste = url_showSavedPath+"?paste="+paste_path+"&num="+parsedmess[0];
+
+    var action_icon_a = document.createElement("A");
+    action_icon_a.setAttribute("TARGET", "_blank");
+    action_icon_a.setAttribute("HREF", url_to_saved_paste);
+    var action_icon_span = document.createElement('SPAN');
+    action_icon_span.className = "fa fa-search-plus";
+    action_icon_a.appendChild(action_icon_span);
+ 
+    inspect.appendChild(action_icon_a);
+
     tr.appendChild(time)
     tr.appendChild(chan);
     tr.appendChild(level);
@@ -177,6 +269,7 @@ function create_log_table(obj_json) {
     tr.appendChild(pdate);
     tr.appendChild(nam);
     tr.appendChild(msage);
+    tr.appendChild(inspect);
 
     if (tr.className == document.getElementById("checkbox_log_info").value && document.getElementById("checkbox_log_info").checked  == true) {
            tableBody.appendChild(tr);
@@ -207,7 +300,7 @@ function create_queue_table() {
     table.appendChild(tableHead);
     table.appendChild(tableBody);
     var heading = new Array();
-    heading[0] = "Queue Name"
+    heading[0] = "Queue Name.PID"
     heading[1] = "Amount"
     var tr = document.createElement('TR');
     tableHead.appendChild(tr);
@@ -219,37 +312,47 @@ function create_queue_table() {
         tr.appendChild(th);
     }
 
-    for(i = 0; i < (glob_tabvar.row1).length;i++){
+    if ((glob_tabvar.row1).length == 0) {
-        var tr = document.createElement('TR')
+        var tr = document.createElement('TR');
-        for(j = 0; j < 2; j++){
+        var td = document.createElement('TD');
-            var td = document.createElement('TD')
+        var td2 = document.createElement('TD');
-            var moduleNum = j == 0 ? "." + glob_tabvar.row1[i][3] : "";
+        td.appendChild(document.createTextNode("No running queues"));
-            td.appendChild(document.createTextNode(glob_tabvar.row1[i][j] + moduleNum));
+        td2.appendChild(document.createTextNode("Or no feed"));
-            tr.appendChild(td)
+        td.className += " danger";
-        }
+        td2.className += " danger";
-        // Used to decide the color of the row
+        tr.appendChild(td);
-        // We have glob_tabvar.row1[][j] with:
+        tr.appendChild(td2);
-        // - j=0: ModuleName
-        // - j=1: queueLength
-        // - j=2: LastProcessedPasteTime
-        // - j=3: Number of the module belonging in the same category
-        if (parseInt(glob_tabvar.row1[i][2]) > 60*2 && parseInt(glob_tabvar.row1[i][1]) > 2)
-            tr.className += " danger";
-        else if (parseInt(glob_tabvar.row1[i][2]) > 60*1)
-            tr.className += " warning";
-        else
-            tr.className += " success";
         tableBody.appendChild(tr);
     }
+    else {
+        for(i = 0; i < (glob_tabvar.row1).length;i++){
+            var tr = document.createElement('TR')
+            for(j = 0; j < 2; j++){
+                var td = document.createElement('TD')
+                var moduleNum = j == 0 ? "." + glob_tabvar.row1[i][3] : "";
+                td.appendChild(document.createTextNode(glob_tabvar.row1[i][j] + moduleNum));
+                tr.appendChild(td)
+            }
+            // Used to decide the color of the row
+            // We have glob_tabvar.row1[][j] with:
+            // - j=0: ModuleName
+            // - j=1: queueLength
+            // - j=2: LastProcessedPasteTime
+            // - j=3: Number of the module belonging in the same category
+            if (parseInt(glob_tabvar.row1[i][2]) > window.threshold_stucked_module && parseInt(glob_tabvar.row1[i][1]) > 2)
+                tr.className += " danger";
+            else if (parseInt(glob_tabvar.row1[i][1]) == 0)
+                tr.className += " warning";
+            else
+                tr.className += " success";
+            tableBody.appendChild(tr);
+        }
+    }
     Tablediv.appendChild(table);
 }
 
-$(document).ready(function () {
-    if (typeof glob_tabvar == "undefined")
-        location.reload();
-    if (typeof glob_tabvar.row1 == "undefined")
-        location.reload();
 
+function load_queues() {
     var data = [];
     var data2 = [];
     var tmp_tab = [];
@@ -259,13 +362,17 @@ $(document).ready(function () {
     var x = new Date();
 
     for (i = 0; i < glob_tabvar.row1.length; i++){
-        if (glob_tabvar.row1[i][0] == 'Categ' || glob_tabvar.row1[i][0] == 'Curve'){
+        if (glob_tabvar.row1[i][0].split(".")[0] == 'Categ' || glob_tabvar.row1[i][0].split(".")[0] == 'Curve'){
-            tmp_tab2.push(0);
+            if (curves_labels2.indexOf(glob_tabvar.row1[i][0].split(".")[0]) == -1) {
-            curves_labels2.push(glob_tabvar.row1[i][0]);
+                tmp_tab2.push(0);
+                curves_labels2.push(glob_tabvar.row1[i][0].split(".")[0]);
+            }
         }
         else {
-            tmp_tab.push(0);
+            if (curves_labels.indexOf(glob_tabvar.row1[i][0].split(".")[0]) == -1) {
-            curves_labels.push(glob_tabvar.row1[i][0]);
+                tmp_tab.push(0);
+                curves_labels.push(glob_tabvar.row1[i][0].split(".")[0]);
+            }
         }
     }
     tmp_tab.unshift(x);
@@ -324,19 +431,29 @@ $(document).ready(function () {
                     update_values();
 
                     if($('#button-toggle-queues').prop('checked')){
+                        $("#queue-color-legend").show();
                         create_queue_table();
                     }
                     else{
                         $("#queueing").html('');
+                        $("#queue-color-legend").hide();
                     }
 
 
+                    queues_pushed = []
                     for (i = 0; i < (glob_tabvar.row1).length; i++){
-                        if (glob_tabvar.row1[i][0] == 'Categ' || glob_tabvar.row1[i][0] == 'Curve'){
+                        if (glob_tabvar.row1[i][0].split(".")[0] == 'Categ' || glob_tabvar.row1[i][0].split(".")[0] == 'Curve'){
-                            tmp_values2.push(glob_tabvar.row1[i][1]);
+                            if (queues_pushed.indexOf(glob_tabvar.row1[i][0].split(".")[0]) == -1) {
+                                queues_pushed.push(glob_tabvar.row1[i][0].split(".")[0]);
+                                tmp_values2.push(parseInt(glob_tabvar.row1[i][1]));
+                            }
                         }
                         else {
-                            tmp_values.push(glob_tabvar.row1[i][1]);
+                            if (queues_pushed.indexOf(glob_tabvar.row1[i][0].split(".")[0]) == -1) {
+                                queues_pushed.push(glob_tabvar.row1[i][0].split(".")[0]);
+                                tmp_values.push(parseInt(glob_tabvar.row1[i][1]));
+                            }
+                            
                         }
                     }
                     tmp_values.unshift(x);
@@ -375,7 +492,19 @@ $(document).ready(function () {
       // something went wrong, hide the canvas container
       document.getElementById('myCanvasContainer').style.display = 'none';
     }
+}
 
+function manage_undefined() {
+    if (typeof glob_tabvar == "undefined")
+        setTimeout(function() { if (typeof glob_tabvar == "undefined") { manage_undefined(); } else { load_queues(); } }, 1000);
+    else if (typeof glob_tabvar.row1 == "undefined")
+        setTimeout(function() { if (typeof glob_tabvar.row1 == "undefined") { manage_undefined(); } else { load_queues(); } }, 1000);
+    else
+        load_queues();
+}
+
+$(document).ready(function () {
+    manage_undefined();
 });
 
 

var/www/templates/browse_important_paste.html

@@ -30,9 +30,6 @@
          white-space:pre-wrap;
          word-wrap:break-word;
       }
-      .modal-backdrop.fade {
-         opacity: 0;
-      }
   </style>
 
 </head>

var/www/templates/important_paste_by_module.html

@@ -1,4 +1,4 @@
-<table class="table table-striped table-bordered table-hover" id="myTable">
+<table class="table table-striped table-bordered table-hover" id="myTable_{{ moduleName }}">
     <thead>
          <tr>
             <th>#</th>
@@ -23,12 +23,117 @@
     </tbody>
 </table>
 </br>
+<div id="nbr_entry" class="alert alert-info">
+ 
+</div>
+<div id="div_stil_data">
+    <button id="load_more_json_button1" type="button" class="btn btn-default" onclick="add_entries(100)" style="display: none">Load 100 entries</button>
+    <button id="load_more_json_button2" type="button" class="btn btn-warning" onclick="add_entries(300)" style="display: none">Load 300 entries</button>
+    <img id="loading_gif_browse" src="{{url_for('static', filename='image/loading.gif') }}" heigt="20" width="20" style="margin: 2px;"></div>
 </br>
 
+
+
 <script>
+var json_array = [];
+var all_data_received = false;
+var curr_numElem;
+var elem_added = 10; //10 elements are added by default in the page loading
+var tot_num_entry = 10; //10 elements are added by default in the page loading
+
+function deploy_source() {
+    var button_load_more_displayed = false;
+
+    if(typeof(EventSource) !== "undefined" && typeof(source) !== "") {
+        $("#load_more_json_button1").show();
+        $("#load_more_json_button2").show();
+        var source = new EventSource("{{ url_for('getImportantPasteByModule') }}"+"?moduleName="+moduleName);
+        source.onmessage = function(event) {
+            var feed = jQuery.parseJSON( event.data );
+            curr_numElem = parseInt($("#myTable_"+moduleName).attr('data-numElem'));
+            if (feed.index > curr_numElem & feed.module == moduleName) { // Avoid doubling the pastes
+                json_array.push(feed);
+                tot_num_entry++;
+                $("#nbr_entry").text(tot_num_entry + " entries available, " + (tot_num_entry - elem_added) + " not displayed");
+
+                $("#myTable_"+moduleName).attr('data-numElem', curr_numElem+1);
+
+                if(feed.index > 100 && !button_load_more_displayed) {
+                    button_load_more_displayed = true;
+                    add_entries_X(20);
+                }
+                if(feed.finished) {
+                    $("#loading_gif_browse").hide();
+                    source.close();
+                    all_data_received = true;
+                    add_entries_X(10);
+                }
+            }
+        };
+    } else {
+        console.log("Sorry, your browser does not support server-sent events...");
+    }
+}
+
+function add_entries(iter) { //Used to disable the button before going to the big loop
+    $("#load_more_json_button1").attr('disabled','disabled');
+    $("#load_more_json_button2").attr('disabled','disabled');
+    setTimeout(function() { add_entries_X(iter);}, 50);
+}
+
+function add_entries_X(to_add) {
+    for(i=0; i<to_add; i++) {
+        if(json_array.length == 0 && all_data_received) {
+            $("#load_more_json_button1").hide();
+            $("#load_more_json_button2").hide();
+            $("#nbr_entry").hide();
+            return false;
+        } else {
+            var feed = json_array.shift();
+            elem_added++;
+             search_table.row.add( [
+                 feed.index,
+                "<a target=\"_blank\" href=\"{{ url_for('showsavedpaste') }}?paste="+feed.path+"&num="+feed.index+"\"> "+ feed.path +"</a>",
+                 feed.date,
+                 feed.linenum,
+                 "<p><span class=\"glyphicon glyphicon-info-sign\" data-toggle=\"tooltip\" data-placement=\"left\" title=\""+feed.content.replace(/\"/g, "\'").replace(/\r/g, "\'").replace(/\n/g, "\'")+"\"></span> <button type=\"button\" class=\"btn-link\" data-num=\""+feed.index+"\" data-toggle=\"modal\" data-target=\"#mymodal\" data-url=\"{{ url_for('showsavedpaste') }}?paste="+feed.path+"&num="+feed.index+"\" data-path=\""+feed.path+"\"><span class=\"fa fa-search-plus\"></span></button></p>"
+             ] ).draw( false );
+             $("#myTable_"+moduleName).attr('data-numElem', curr_numElem+1);
+        }
+    }
+    $("#load_more_json_button1").removeAttr('disabled');
+    $("#load_more_json_button2").removeAttr('disabled');
+    $("#nbr_entry").text(tot_num_entry + " entries available, " + (tot_num_entry - elem_added) + " not displayed");
+    return true;
+}
+
+
+
+
+</script>
+
+
+<script>
+var moduleName = "{{ moduleName }}";
+var search_table;
+var last_clicked_paste;
+var can_change_modal_content = true;
+$("#myTable_"+moduleName).attr('data-numElem', "{{ all_path|length }}");
+
 $(document).ready(function(){
     $('[data-toggle="tooltip"]').tooltip();
-    $('#myTable').dataTable({ "order": [[ 2, "desc" ]] });
+
+    search_table = $('#myTable_'+moduleName).DataTable({ "order": [[ 2, "desc" ]] });
+
+    if( "{{ finished }}" == "True"){
+       $("#load_more_json_button1").hide();
+       $("#load_more_json_button2").hide();
+       $("#nbr_entry").hide();
+       $("#loading_gif_browse").hide();
+    } else {
+        deploy_source();
+    }
+
 });
 </script>
 
@@ -40,39 +145,9 @@ $(document).ready(function(){
     var char_to_display = {{ char_to_display }};
     var start_index = 0;
 
-    // On click, get html content from url and update the corresponding modal
-    $("[data-toggle='modal']").on("click.openmodal", function (event) {
-        event.preventDefault();
-        var modal=$(this);
-        var url = " {{ url_for('showpreviewpaste')  }}?paste=" + $(this).attr('data-path') + "&num=" + $(this).attr('data-num');
-        $.get(url, function (data) {
-            $("#mymodalbody").html(data);
-            var button = $('<button type="button" id="load-more-button" class="btn btn-info btn-xs center-block" data-url="' + $(modal).attr('data-path') +'" data-toggle="tooltip" data-placement="bottom" title="Load more content"><span class="glyphicon glyphicon-download"></span></button>');
-            button.tooltip();
-            $("#mymodalbody").children(".panel-default").append(button);
-
-            $("#button_show_path").attr('href', $(modal).attr('data-url'));
-            $("#button_show_path").show('fast');
-            $("#loading-gif-modal").css("visibility", "hidden"); // Hide the loading GIF
-            if ($("[data-initsize]").attr('data-initsize') < char_to_display) { // All the content is displayed
-                nothing_to_display();
-            }
-            // On click, donwload all paste's content
-            $("#load-more-button").on("click", function (event) {
-                if (complete_paste == null) { //Donwload only once
-                    $.get("{{ url_for('getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){
-                        complete_paste = data;
-                        update_preview();
-                    });
-                } else {
-                    update_preview();
-                }
-            });
-        });
-    });
-
     // When the modal goes out, refresh it to normal content
     $("#mymodal").on('hidden.bs.modal', function () {
+        can_change_modal_content = true;
         $("#mymodalbody").html("<p>Loading paste information...</p>");
         var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_for('static', filename='image/loading.gif') }}\" height='26' width='26' style='margin: 4px;'>";
         $("#mymodalbody").append(loading_gif); // Show the loading GIF
@@ -110,38 +185,53 @@ $(document).ready(function(){
    }
 
 
-
    // Use to bind the button with the new displayed data
    // (The bind do not happens if the dataTable is in tabs and the clicked data is in another page)
-   $('#myTable').on( 'draw.dt', function () {
+
+   search_table.on( 'draw.dt', function () {
+       // Bind tooltip each time we draw a new page
+       $('[data-toggle="tooltip"]').tooltip();
        // On click, get html content from url and update the corresponding modal
-       $("[data-toggle='modal']").unbind('click.openmodal').on("click.openmodal", function (event) {
+       $("[data-toggle='modal']").off('click.openmodal').on("click.openmodal", function (event) {
            event.preventDefault();
            var modal=$(this);
            var url = " {{ url_for('showpreviewpaste')  }}?paste=" + $(this).attr('data-path') + "&num=" + $(this).attr('data-num');
+           last_clicked_paste =  $(this).attr('data-num');
            $.get(url, function (data) {
-               $("#mymodalbody").html(data);
-               var button = $('<button type="button" id="load-more-button" class="btn btn-info btn-xs center-block" data-url="' + $(modal).attr('data-path') +'" data-toggle="tooltip" data-placement="bottom" title="Load more content"><span class="glyphicon glyphicon-download"></span></button>');
-               button.tooltip();
-               $("#mymodalbody").children(".panel-default").append(button);
 
-               $("#button_show_path").attr('href', $(modal).attr('data-url'));
+               // verify that the reveived data is really the current clicked paste. Otherwise, ignore it. 
-               $("#button_show_path").show('fast');
+               var received_num = parseInt(data.split("|num|")[1]);
-               $("#loading-gif-modal").css("visibility", "hidden"); // Hide the loading GIF
+               if (received_num == last_clicked_paste && can_change_modal_content) {
-               if ($("[data-initsize]").attr('data-initsize') < char_to_display) { // All the content is displayed
+                   can_change_modal_content = false;
-                   nothing_to_display();
+
-               }
+                   // clear data by removing html, body, head tags. prevent dark modal background stack bug.
-               // On click, donwload all paste's content
+                   var cleared_data = data.split("<body>")[1].split("</body>")[0];
-               $("#load-more-button").on("click", function (event) {
+                   $("#mymodalbody").html(cleared_data);
-                   if (complete_paste == null) { //Donwload only once
+
-                       $.get("{{ url_for('getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){
+                   var button = $('<button type="button" id="load-more-button" class="btn btn-info btn-xs center-block" data-url="' + $(modal).attr('data-path') +'" data-toggle="tooltip" data-placement="bottom" title="Load more content"><span class="glyphicon glyphicon-download"></span></button>');
-                           complete_paste = data;
+                   button.tooltip();
-                           update_preview();
+                   $("#mymodalbody").children(".panel-default").append(button);
-                       });
+       
-                   } else {
+                   $("#button_show_path").attr('href', $(modal).attr('data-url'));
-                       update_preview();
+                   $("#button_show_path").show('fast');
+                   $("#loading-gif-modal").css("visibility", "hidden"); // Hide the loading GIF
+                   if ($("[data-initsize]").attr('data-initsize') < char_to_display) { // All the content is displayed
+                       nothing_to_display();
                    }
-               });
+                   // On click, donwload all paste's content
+                   $("#load-more-button").on("click", function (event) {
+                       if (complete_paste == null) { //Donwload only once
+                           $.get("{{ url_for('getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){
+                               complete_paste = data;
+                               update_preview();
+                           });
+                       } else {
+                           update_preview();
+                       }
+                   });
+               } else if (can_change_modal_content) {
+                   $("#mymodalbody").html("Ignoring previous not finished query of paste #" + received_num);
+               }
            });
        });
    } );

var/www/templates/index.html

@@ -17,9 +17,11 @@
   <script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
   <script src="{{ url_for('static', filename='js/jquery.js') }}"></script>
   <script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
   <script>
       window.default_minute = {{ default_minute }};
       window.glob_tabvar = []; // Avoid undefined
+      window.threshold_stucked_module = {{ threshold_stucked_module }}
       function update_values() {
           $SCRIPT_ROOT = {{ request.script_root|tojson|safe }};
           $.getJSON($SCRIPT_ROOT+"/_stuff",
@@ -51,10 +53,10 @@
               <!-- /.sidebar-collapse -->
               <div class="panel panel-default">
                   <div class="panel-heading">
-                      <i class="fa fa-dashboard  fa-fw"></i> Pastes since {{ default_minute }} min
+                      <i class="fa fa-dashboard  fa-fw"></i> Total pastes since {{ default_minute }} min
                   </div>
                   <div class="panel-body">
-                       <div id="realtimechart" style="height: 90px; padding: 0px; position: relative;"></div>
+                       <div id="global" style="height: 90px; padding: 0px; position: relative;"></div>
                   </div>
               </div>
                   <!-- <div id="Graph_paste_num" style="height:90px; width:100%;"></div> -->
@@ -66,7 +68,18 @@
                   </label>
                   <strong style="top: 3px; position: relative;">Display queues</strong>
               <div>
-              <div class="table-responsive", id="queueing" style="margin-top:10px;"></div>
+              <div>
+                  <table id="queue-color-legend" class="table">
+                  <thead>
+                  </thead>
+                  <tbody>
+                      <tr><td class="legendColorBox" style="vertical-align: ; "><div style="border:1px solid #ccc;padding:1px"><div style="width:100%;height:0;border:5px solid #d0e9c6;overflow:hidden"></div></div></td><td> Working queues</td></tr>
+                      <tr><td class="legendColorBox" style="vertical-align: ;"><div style="border:1px solid #ccc;padding:1px"><div style="width:100%;height:0;border:5px solid #faf2cc;overflow:hidden"></div></div></td><td> Idling queues</td></tr>
+                      <tr><td class="legendColorBox" style="vertical-align: ;"><div style="border:1px solid #ccc;padding:1px"><div style="width:100%;height:0;border:5px solid #ebcccc;overflow:hidden"></div></div></td><td> Stuck queues</td></tr>
+                  </tbody>
+                  </table>
+              </div>
+              <div class="table-responsive", id="queueing" style="margin-top:10px; font-size: small;"></div>
              <a  href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
           </div>
           <!-- /.navbar-static-side -->
@@ -79,23 +92,43 @@
         </div>
 
             <div class="row">
-                      <div class="col-lg-12">
+                 <div class="col-lg-6">
-                      <div class="panel panel-default">
+                     <div class="panel panel-default">
-                      <div class="panel-heading">
+                        <div class="panel-heading">
-                        <i class="fa fa-bar-chart-o fa-fw"></i> Queues Monitor
+                           <i class="fa fa-bar-chart-o fa-fw"></i> Feeder(s) Monitor:
-                      </div>
-                        <div class="panel-body">
-                            <div class="row">
-                                <div class="col-lg-6" id="Graph" style="height:400px; width:48%;"></div>
-                                <div class="col-lg-6" id="Graph2" style="height:400px; width:48%;"></div>
-                            </div>
-                            <!-- /.row -->
                         </div>
-                        <!-- /.panel-body -->
+                        <div id="panelbody" class="panel-body" style="height:420px;">
-                    </div>
+
-                    <!-- /.panel -->
+                       <strong>Processed pastes</strong>
-                </div>
+                       <div id="Proc_feeder" style="height: 230px; padding: 0px; position: relative;"></div>
-                <!-- /.col-lg-8 -->
+                       <hr style="border-top: 2px solid #eee; margin-top: 7px; margin-bottom: 7px;">
+                       <strong>Filtered duplicated</strong>
+                       <div id="Dup_feeder" style="height: 100px; padding: 0px; position: relative;"></div>
+
+                        </div>
+                     <!-- /.panel-body -->
+                     </div>
+                      <!-- /.panel -->
+                 </div>
+                 <!-- /.col-lg-6 -->
+                 <div class="col-lg-6">
+                     <div class="panel panel-default">
+                        <div class="panel-heading">
+                           <i class="fa fa-bar-chart-o fa-fw"></i> Queues Monitor
+                        </div>
+                        <div class="panel-body">
+                           <div class="row">
+                               <div class="col-lg-6" id="Graph" style="height:195px; width:88%;"></div>
+                               <div style="height:10px;"></div>
+                               <div class="col-lg-6" id="Graph2" style="height:195px; width:88%;"></div>
+                           </div>
+                        <!-- /.row -->
+                        </div>
+                     <!-- /.panel-body -->
+                     </div>
+                      <!-- /.panel -->
+                 </div>
+                 <!-- /.col-lg-6 -->
         <div class="col-lg-12">
         <div class="panel panel-default">
         <div class="panel-heading">
@@ -129,6 +162,7 @@
 						<th>Date</th>
 						<th>Paste name</th>
 						<th>Message</th>
+						<th>Actions</th>
 						</tr>
 					</thead>
 					<tbody id="tab_body">
@@ -142,6 +176,7 @@
         <!-- /#page-wrapper -->
 
     </div>
+    <script> var url_showSavedPath = "{{ url_for('showsavedpaste') }}"; </script>
     <script type="text/javascript" src="{{ url_for('static', filename='js/indexjavascript.js')}}"></script>
 
     <script>

var/www/templates/search.html

@@ -31,9 +31,7 @@
          white-space:pre-wrap;
          word-wrap:break-word;
       }
-      .modal-backdrop.fade {
+
-         opacity: 0;
-      }
   </style>
 
 </head>
@@ -69,7 +67,7 @@
             </div>
             <div class="modal-footer">
               <a id="button_show_path" target="_blank" href=""><button type="button" class="btn btn-info">Show saved paste</button></a>
-              <button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
+              <button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
             </div>
           </div>
         </div>
@@ -84,7 +82,7 @@
                     </br>
                     <div class="panel panel-default">
                         <div class="panel-heading">
-                            <i class="glyphicon glyphicon-search"></i> {{ r|length }} Results for "<strong>{{ query }}</strong>"
+                            <i class="glyphicon glyphicon-search"></i> <b id="numberOfRes">{{ r|length }}</b> Results for "<strong>{{ query }}</strong>
                             <div class="pull-right">
 
                             </div>
@@ -101,7 +99,7 @@
                                       <th>Action</th>
                                    </tr>
                               </thead>
-                              <tbody>
+                              <tbody id="table_body">
                           {% set i = 0 %}
                           {% for path in r %}
                                 <tr>
@@ -115,7 +113,11 @@
                           {% endfor %}
                               </tbody>
                           </table>
+                          <div id="div_stil_data">
+                              <button id="load_more_json_button1" type="button" class="btn btn-default" onclick="add_entries();" style="display: none">Load 50 entries</button>
+                              <strong> Totalling {{ num_res }} results related to paste content </strong>
                           </div>
+                        </div>
                         <!-- /.panel-body -->
                     </div>
                 </div>
@@ -124,15 +126,78 @@
         <!-- /#page-wrapper -->
     </div>
     <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
+</br>
 </body>
 
 <!-- enable tooltip and dataTable -->
 <script>
+    var search_table;
+    var last_clicked_paste;
+    var can_change_modal_content = true;
+    var page_offset;
+    var offset;
+    var all_loaded;
+    var init_num_of_elements_in_table;
+    var query;
+    var pagelen = 50;
+
     $(document).ready(function(){
         $('[data-toggle="tooltip"]').tooltip();
+
         $("#button_show_path").hide();
-	$('#myTable').dataTable();
+        search_table = $('#myTable').DataTable();
+
+        query = "{{ query }}";
+        offset = 0;
+        page_offset = 2; //page 1 already loaded
+        all_loaded = false;
+        init_num_of_elements_in_table = parseInt("{{ r|length }}"); // Comes from the file search
+
+
+        if (init_num_of_elements_in_table == pagelen) {
+            $("#load_more_json_button1").show();
+        }
     });
+
+</script>
+
+<script>
+
+// Loop to recover all the data from  get_more_search_results
+// And add it dynamically top the dataTable
+    function add_entries() { //Used to disable the button before going to the big loop
+        $("#load_more_json_button1").attr('disabled','disabled');
+        setTimeout(function() { load_search_50_data();}, 50);
+    }
+
+    function load_search_50_data() {
+        var options = { query: query, page_offset: page_offset };
+        $.post( "{{ url_for('get_more_search_result') }}", options).done(function( data ) {
+
+            for(i=0; i<data.path_array.length; i++) {
+                var curr_preview = data.preview_array[i].replace(/\"/g, "\'");
+                search_table.row.add( [
+                    init_num_of_elements_in_table+((offset))+i+1,
+                   "<a target=\"_blank\" href=\"{{ url_for('showsavedpaste') }}?paste="+data.path_array[i]+"&num="+i+"\"> "+ data.path_array[i] +"</a>",
+                    data.date_array[i],
+                    data.size_array[i],
+                    "<p><span class=\"glyphicon glyphicon-info-sign\" data-toggle=\"tooltip\" data-placement=\"left\" title=\""+curr_preview+"\"></span> <button type=\"button\" class=\"btn-link\" data-num=\""+i+"\" data-toggle=\"modal\" data-target=\"#mymodal\" data-url=\"{{ url_for('showsavedpaste') }}?paste="+data.path_array[i]+"&num="+i+"\" data-path=\""+data.path_array[i]+"\"><span class=\"fa fa-search-plus\"></span></button></p>"
+                ] ).draw( false );
+            }
+            offset = offset + data.path_array.length;
+            page_offset = page_offset+1;
+            $("#numberOfRes").text(parseInt($("#numberOfRes").text()) + data.path_array.length);
+            if (data.moreData == true) {
+                //continue
+            } else {
+                all_loaded = true;
+                $("#load_more_json_button1").hide();
+            }
+            $("#load_more_json_button1").removeAttr('disabled');
+            return data.path_array.length;
+        });
+    }
+
 </script>
 
 <!-- Dynamically update the modal -->
@@ -143,39 +208,9 @@
     var char_to_display = {{ char_to_display }};
     var start_index = 0;
 
-    // On click, get html content from url and update the corresponding modal
-    $("[data-toggle='modal']").on("click", function (event) {
-        event.preventDefault();
-        var modal=$(this);
-        var url = " {{ url_for('showpreviewpaste')  }}?paste=" + $(this).attr('data-path') + "&num=" + $(this).attr('data-num');
-        $.get(url, function (data) {
-            $("#mymodalbody").html(data);
-            var button = $('<button type="button" id="load-more-button" class="btn btn-info btn-xs center-block" data-url="' + $(modal).attr('data-path') +'" data-toggle="tooltip" data-placement="bottom" title="Load more content"><span class="glyphicon glyphicon-download"></span></button>');
-            button.tooltip();
-            $("#mymodalbody").children(".panel-default").append(button);
-
-            $("#button_show_path").attr('href', $(modal).attr('data-url'));
-            $("#button_show_path").show('fast');
-            $("#loading-gif-modal").css("visibility", "hidden"); // Hide the loading GIF
-            if ($("[data-initsize]").attr('data-initsize') < char_to_display) { // All the content is displayed
-                nothing_to_display();
-            }
-            // On click, donwload all paste's content
-            $("#load-more-button").on("click", function (event) {
-                if (complete_paste == null) { //Donwload only once
-                    $.get("{{ url_for('getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){
-                        complete_paste = data;
-                        update_preview();
-                    });
-                } else {
-                    update_preview();
-                }
-            });
-        });
-    });
-
     // When the modal goes out, refresh it to normal content
     $("#mymodal").on('hidden.bs.modal', function () {
+        can_change_modal_content = true;
         $("#mymodalbody").html("<p>Loading paste information...</p>");
         var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_for('static', filename='image/loading.gif') }}\" height='26' width='26' style='margin: 4px;'>";
         $("#mymodalbody").append(loading_gif); // Show the loading GIF
@@ -211,5 +246,55 @@
        new_content.show('fast');
        $("#load-more-button").hide();
    }
+
+
+   $('#myTable').on( 'draw.dt', function () {
+       // Bind tooltip each time we draw a new page
+       $('[data-toggle="tooltip"]').tooltip();
+       // On click, get html content from url and update the corresponding modal
+       $("[data-toggle='modal']").off('click.openmodal').on("click.openmodal", function (event) {
+           var modal=$(this);
+           var url = " {{ url_for('showpreviewpaste')  }}?paste=" + $(this).attr('data-path') + "&num=" + $(this).attr('data-num');
+           last_clicked_paste =  $(this).attr('data-num');
+           $.get(url, function (data) {
+
+               // verify that the reveived data is really the current clicked paste. Otherwise, ignore it. 
+               var received_num = parseInt(data.split("|num|")[1]);
+               if (received_num == last_clicked_paste && can_change_modal_content) {
+                   can_change_modal_content = false;
+
+                   // clear data by removing html, body, head tags. prevent dark modal background stack bug.
+                   var cleared_data = data.split("<body>")[1].split("</body>")[0];
+                   $("#mymodalbody").html(cleared_data);
+                   setTimeout(function() { $('#tableDup').DataTable(); }, 150);
+
+                   var button = $('<button type="button" id="load-more-button" class="btn btn-info btn-xs center-block" data-url="' + $(modal).attr('data-path') +'" data-toggle="tooltip" data-placement="bottom" title="Load more content"><span class="glyphicon glyphicon-download"></span></button>');
+                   button.tooltip();
+                   $("#mymodalbody").children(".panel-default").append(button);
+   
+                   $("#button_show_path").attr('href', $(modal).attr('data-url'));
+                   $("#button_show_path").show('fast');
+                   $("#loading-gif-modal").css("visibility", "hidden"); // Hide the loading GIF
+                   if ($("[data-initsize]").attr('data-initsize') < char_to_display) { // All the content is displayed
+                       nothing_to_display();
+                   }
+                   // On click, donwload all paste's content
+                   $("#load-more-button").off('click.download').on("click.download", function (event) {
+                       if (complete_paste == null) { //Donwload only once
+                           $.get("{{ url_for('getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){
+                               complete_paste = data;
+                               update_preview();
+                           });
+                       } else {
+                           update_preview();
+                       }
+                   });
+               } else if (can_change_modal_content) {
+                   $("#mymodalbody").html("Ignoring previous not finished query of paste #" + received_num);
+               }
+           });
+       });
+   } );
+
 </script>
 </html>

var/www/templates/sentiment_analysis_trending.html

@@ -205,24 +205,24 @@
                                         </thead>
                                         <tbody>
                                           <tr>
-                                              <td class="worst1 moodtable_worst">worst1</td>
+                                              <td class="worst1 moodtable_worst">no data</td>
-                                              <td class="best1 moodtable_best">best1</td>
+                                              <td class="best1 moodtable_best">no data</td>
                                           </tr>
                                           <tr>
-                                              <td class="worst2 moodtable_worst">worst2</td>
+                                              <td class="worst2 moodtable_worst">no data</td>
-                                              <td class="best2 moodtable_best">best2</td>
+                                              <td class="best2 moodtable_best">no data</td>
                                           </tr>
                                           <tr>
-                                              <td class="worst3 moodtable_worst">worst3</td>
+                                              <td class="worst3 moodtable_worst">no data</td>
-                                              <td class="best3 moodtable_best">best3</td>
+                                              <td class="best3 moodtable_best">no data</td>
                                           </tr>
                                           <tr>
-                                              <td class="worst4 moodtable_worst">worst4</td>
+                                              <td class="worst4 moodtable_worst">no data</td>
-                                              <td class="best4 moodtable_best">best4</td>
+                                              <td class="best4 moodtable_best">no data</td>
                                           </tr>
                                           <tr>
-                                              <td class="worst5 moodtable_worst">worst5</td>
+                                              <td class="worst5 moodtable_worst">no data</td>
-                                              <td class="best5 moodtable_best">best5</td>
+                                              <td class="best5 moodtable_best">no data</td>
                                           </tr>
                                         </tbody>
                                     </table>

var/www/templates/show_saved_paste.html

@@ -4,68 +4,87 @@
   <title>Paste information</title>
   <meta charset="utf-8">
   <meta name="viewport" content="width=device-width, initial-scale=1">
+
+   <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
+   <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
+   <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
+   <link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
+   <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+   <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
+   <script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
+   <script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
+   <script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
+   <script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
+   <script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>     
 </head>
 <body>
+    <div class="panel panel-default">
+      <div class="panel-heading">
+        <h1 class="page-header" >Paste: {{ request.args.get('paste') }}</h1>
+        <h2 class="page-header" >({{ request.args.get('num') }})</h2>
   
-  <h2> Paste: {{ request.args.get('num') }}</h2>
+        <table class="table table-condensed">
-  <h3> {{ request.args.get('paste') }} </h3>
+          <thead>
-
+            <tr>
-  <hr></br>
+              <th>Date</th>
-
+              <th>Source</th>
-<div class="panel panel-default">
+              <th>Encoding</th>
-  <div class="panel-heading">
+              <th>Language</th>
-    <table class="table table-condensed">
+              <th>Size (Kb)</th>
-      <thead>
+              <th>Mime</th>
-        <tr>
+              <th>Number of lines</th>
-          <th>Date</th>
+              <th>Max line length</th>
-          <th>Source</th>
+            </tr>
-          <th>Encoding</th>
+          </thead>
-          <th>Language</th>
+          <tbody>
-          <th>Size (Kb)</th>
+            <tr>
-          <th>Mime</th>
+            <td>{{ date }}</td>
-          <th>Number of lines</th>
+            <td>{{ source }}</td>
-          <th>Max line length</th>
+            <td>{{ encoding }}</td>
-        </tr>
+            <td>{{ language }}</td>
-      </thead>
+            <td>{{ size }}</td>
-      <tbody>
+            <td>{{ mime }}</td>
-        <tr>
+            <td>{{ lineinfo.0 }}</td>
-        <td>{{ date }}</td>
+            <td>{{ lineinfo.1 }}</td>
-        <td>{{ source }}</td>
+            </tr>
-        <td>{{ encoding }}</td>
+          </tbody>
-        <td>{{ language }}</td>
+        </table>
-        <td>{{ size }}</td>
+      </div>
-        <td>{{ mime }}</td>
+    <div class="panel-body" id="panel-body">
-        <td>{{ lineinfo.0 }}</td>
+      {% if  duplicate_list|length == 0 %}
-        <td>{{ lineinfo.1 }}</td>
+          <h3> No Duplicate </h3>
-        </tr>
+      {% else %}
-      </tbody>
+          <h3> Duplicate list: </h3>
-    </table>
+          <table id="tableDup" class="table table-striped table-bordered">
-  </div>
+          {% set i = 0 %}
-  <div class="panel-body" id="panel-body">
+          <thead>
-  {% if  duplicate_list|length == 0 %}
-      <h3> No Duplicate </h3>
-  {% else %}
-      <h3> Duplicate list: </h3>
-      <table style="width:100%">
-      {% set i = 0 %}
-      <tr>
-          <th style="text-align:left;">Hash type</th><th style="text-align:left;">Paste info</th>
-      </tr>
-      {% for dup_path in duplicate_list %}
           <tr>
-              <td>{{ hashtype_list[i] }}</td>
+              <th>Hash type</th>
-              <td>Similarity: {{ simil_list[i] }}%</td>
+              <th>Paste info</th>
-              <td><a target="_blank" href="{{ url_for('showsavedpaste') }}?paste={{ dup_path }}" id='dup_path'>{{ dup_path }}</a></td>
+              <th>Date</th>
+              <th>Path</th> 
           </tr>
-          {% set i = i + 1 %}
+          </thead>
-      {% endfor %}
+          <tbody>
-      </table>
+          {% for dup_path in duplicate_list %}
-  {% endif %}
+              <tr>
-    <h3> Content: </h3>
+                  <td>{{ hashtype_list[i] }}</td>
-  <p data-initsize="{{ initsize }}"> <pre id="paste-holder">{{ content }}</pre></p>
+                  <td>Similarity: {{ simil_list[i] }}%</td>
-  </div>
+                  <td>{{ date_list[i] }}</td>
-</div>
+                  <td><a target="_blank" href="{{ url_for('showsavedpaste') }}?paste={{ dup_path }}" id='dup_path'>{{ dup_path }}</a></td>
-
+              </tr>
+              {% set i = i + 1 %}
+          {% endfor %}
+          </tbody>
+          </table>
+      {% endif %}
+        <h3> Content: </h3>
+        <p data-initsize="{{ initsize }}"> <pre id="paste-holder">{{ content }}</pre></p>
+      </div>
+    </div>
   
 </body>
+<script>
+    $('#tableDup').DataTable();
+</script>
 </html>

var/www/templates/terms_management.html

@@ -12,7 +12,6 @@
   <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
   <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
   <link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
-  <link href="{{ url_for('static', filename='css/jquery-ui.min.css') }}" rel="stylesheet" type="text/css" />
   <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
   <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
   <script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
@@ -112,7 +111,7 @@
                                         <th>Day occurence</th>
                                         <th>Week occurence</th>
                                         <th>Month occurence</th>
-                                        <th># Concerned pastes</th>
+                                        <th># tracked paste</th>
                                         <th>Action</th>
                                      </tr>
                                 </thead>
@@ -199,6 +198,9 @@
 
 <!-- import graph function -->
 <script>
+    var table_track;
+    var table_black;
+
     $(document).ready(function(){
         activePage = $('h1.page-header').attr('data-page');
         $("#"+activePage).addClass("active");
@@ -208,6 +210,14 @@
 	table_track = $('#myTable').DataTable();
 	table_black = $('#myTable2').DataTable();
 
+        table_track.on( 'draw.dt', function () {
+            perform_binding();
+        });
+        table_black.on( 'draw.dt', function () {
+            perform_binding();
+        });
+
+
         $("#followTermInput").keyup(function(event){
             if(event.keyCode == 13){
                 $("#followTermBtn").click();
@@ -255,7 +265,7 @@
                                 html_to_add += "<td>"+curr_data.size+"</td>";
                                 html_to_add += "<td>"+curr_data.lineinfo[0]+"</td>";
                                 html_to_add += "<td>"+curr_data.lineinfo[1]+"</td>";
-                                html_to_add += "<td><div class=\"row\"><button class=\"btn btn-xs btn-default\" data-toggle=\"popover\" data-placement=\"left\" data-content=\""+curr_data.content+"\">Preview content</button><a target=\"_blank\" href=\"{{ url_for('showsavedpaste') }}?paste="+curr_data.path+"&num=0\">  <button type=\"button\" class=\"btn btn-xs btn-info\">Show Paste</button></a></div></td>";
+                                html_to_add += "<td><div class=\"row\"><button class=\"btn btn-xs btn-default\" data-toggle=\"popover\" data-placement=\"left\" data-content=\""+curr_data.content.replace(/\"/g, "\'")+"\">Preview content</button><a target=\"_blank\" href=\"{{ url_for('showsavedpaste') }}?paste="+curr_data.path+"&num=0\">  <button type=\"button\" class=\"btn btn-xs btn-info\">Show Paste</button></a></div></td>";
 
                             html_to_add += "</tr>";
                     }

var/www/templates/terms_plot_tool.html

@@ -72,7 +72,7 @@
                         </div>
                         <div class="panel-body">
                             <div aria-disabled="false" class="slider sliderRange sliderBlue ui-slider ui-slider-horizontal ui-widget ui-widget-content ui-corner-all" style="margin-bottom: 5px;"></div>
-                            <strong>Date:</strong> <input type="text" id="amount" readonly style="border:0; color:#f6931f; font-weight:bold;">
+                            <strong>Date:</strong> <input type="text" id="amount" readonly style="border:0; color:#f6931f; font-weight:bold;"> <button id="plot-month" onclick="replot();" class="btn btn-info" style="float: right;">Replot</button>
 
                             <div class="form-group input-group" style="margin-top: 30px;">
                                 <span class="input-group-addon"><span class="glyphicon glyphicon-stats"></span></span>
@@ -168,6 +168,7 @@
 <script>
 var plot;
 var graph_data = [];
+var plotted_terms = [];
 var graph_options = {
                         series: {
                             lines: { 
@@ -198,6 +199,7 @@ function plotData() {
     $('#plot-btn-add').show("fast");
     var curthis = $(this);
     var term = $('#TermInput').val();
+    plotted_terms = [term]
     var range_start = new Date($( ".sliderRange" ).slider( "values", 0 )).getTime() / 1000;
     var range_end = new Date($( ".sliderRange" ).slider( "values", 1 )).getTime() / 1000;
 
@@ -205,8 +207,8 @@ function plotData() {
         graph_data = [];
         var to_plot = [];
         var curr_data = [];
-        for(i=0; i<data.length; i++) {
+        for(i=1; i<data.length; i++) {
-                curr_data.push([data[i][0]*1000, data[i][1]]);
+            curr_data.push([data[i][0]*1000, data[i][1]]);
         }
         to_plot.push({ data: curr_data, label: term}); 
         graph_data.push({ data: curr_data, label: term});
@@ -234,22 +236,46 @@ function plotData() {
 function addData() {
     var curthis = $(this);
     var term = $('#TermInput').val();
+    plotted_terms.push(term)
     var range_start = new Date($( ".sliderRange" ).slider( "values", 0 )).getTime() / 1000;
     var range_end = new Date($( ".sliderRange" ).slider( "values", 1 )).getTime() / 1000;
 
     $.getJSON("{{ url_for('terms_plot_tool_data') }}", { range_start: range_start, range_end: range_end, term: term }, function(data, status){
-        var to_plot = [];
         var curr_data = [];
-        for(i=0; i<data.length; i++) {
+        for(i=1; i<data.length; i++) {
                 curr_data.push([data[i][0]*1000, data[i][1]]);
         }
-        to_plot.push({ data: curr_data, label: term}); 
         graph_data.push({ data: curr_data, label: term});
         plot = $.plot($("#graph"), graph_data, graph_options);
         $("#TermInput").val("");
     })
 }
 
+
+function replot() {
+    graph_data = [];
+    promises = [];
+    for(i=0; i<plotted_terms.length; i++) {
+        var term = plotted_terms[i];
+        var range_start = new Date($( ".sliderRange" ).slider( "values", 0 )).getTime() / 1000;
+        var range_end = new Date($( ".sliderRange" ).slider( "values", 1 )).getTime() / 1000;
+
+        promises.push($.getJSON("{{ url_for('terms_plot_tool_data') }}", { range_start: range_start, range_end: range_end, term: term }, function(data, status){
+            var curr_data = [];
+            for(i=1; i<data.length; i++) {
+                    curr_data.push([data[i][0]*1000, data[i][1]]);
+            }
+            graph_data.push({ data: curr_data, label: data[0]});
+            $("#TermInput").val("");
+        }))
+    }
+    
+    $.when.apply($, promises).done( function () {
+        plot = $.plot($("#graph"), graph_data, graph_options);
+    });
+    
+}
+
 </script>
 
 

var/www/templates/terms_plot_top.html

@@ -12,7 +12,6 @@
   <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
   <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
   <link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
-  <link href="{{ url_for('static', filename='css/jquery-ui.min.css') }}" rel="stylesheet" type="text/css" />
   <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
   <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
   <script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>

var/www/update_thirdparty.sh

@@ -4,24 +4,28 @@ set -e
 
 wget http://dygraphs.com/dygraph-combined.js -O ./static/js/dygraph-combined.js
 
-SBADMIN_VERSION='1.0.4'
+SBADMIN_VERSION='3.3.7'
 
 rm -rf temp
 mkdir temp
 
 wget https://github.com/BlackrockDigital/startbootstrap-sb-admin/archive/v${SBADMIN_VERSION}.zip -O temp/${SBADMIN_VERSION}.zip
+wget https://github.com/BlackrockDigital/startbootstrap-sb-admin-2/archive/v${SBADMIN_VERSION}.zip -O temp/${SBADMIN_VERSION}-2.zip
 unzip temp/${SBADMIN_VERSION}.zip -d temp/
-mv temp/startbootstrap-sb-admin-${SBADMIN_VERSION} temp/sb-admin-2
+unzip temp/${SBADMIN_VERSION}-2.zip -d temp/
+mv temp/startbootstrap-sb-admin-${SBADMIN_VERSION} temp/sb-admin
+mv temp/startbootstrap-sb-admin-2-${SBADMIN_VERSION} temp/sb-admin-2
 
 rm -rf ./static/js/plugins
-mv temp/sb-admin-2/js/* ./static/js/
+mv temp/sb-admin/js/* ./static/js/
 
 rm -rf ./static/fonts/ ./static/font-awesome/
-mv temp/sb-admin-2/fonts/ ./static/
+mv temp/sb-admin/fonts/ ./static/
-mv temp/sb-admin-2/font-awesome/ ./static/
+mv temp/sb-admin/font-awesome/ ./static/
 
 rm -rf ./static/css/plugins/
-mv temp/sb-admin-2/css/* ./static/css/
+mv temp/sb-admin/css/* ./static/css/
+mv temp/sb-admin-2/dist/css/* ./static/css/
 
 rm -rf temp
 
@@ -39,12 +43,17 @@ wget https://raw.githubusercontent.com/flot/flot/master/jquery.flot.pie.js -O ./
 wget https://raw.githubusercontent.com/flot/flot/master/jquery.flot.time.js -O ./static/js/jquery.flot.time.js
 wget https://raw.githubusercontent.com/flot/flot/master/jquery.flot.stack.js -O ./static/js/jquery.flot.stack.js
 
-#Ressources for sparkline and canvasJS
+#Ressources for sparkline and canvasJS and slider
 wget http://omnipotent.net/jquery.sparkline/2.1.2/jquery.sparkline.min.js -O ./static/js/jquery.sparkline.min.js
+mkdir temp
 wget http://canvasjs.com/fdm/chart/ -O temp/canvasjs.zip
 unzip temp/canvasjs.zip -d temp/
-mkdir temp
 mv temp/jquery.canvasjs.min.js ./static/js/jquery.canvasjs.min.js
+
+wget https://jqueryui.com/resources/download/jquery-ui-1.12.0.zip -O temp/jquery-ui.zip
+unzip temp/jquery-ui.zip -d temp/
+mv temp/jquery-ui-1.12.0/jquery-ui.min.js ./static/js/jquery-ui.min.js
+mv temp/jquery-ui-1.12.0/jquery-ui.min.css ./static/css/jquery-ui.min.css
 rm -rf temp
 
 mkdir -p ./static/image