Bug fix related with redis:

Fixed typo key in redis for module creditcard and sqlinjection
Modified Curve redisLvlDb server
Modified Url.py so that it forwards name of protocol from saved protocolsfile
Added Cache control in Flask
Modified key-tab name into keys-tab

bin/CreditCard.py

@@ -68,7 +68,7 @@ if __name__ == "__main__":
                     #Send to duplicate
                     p.populate_set_out(filepath, 'Redis_Duplicate')
                     #send to Browse_warning_paste
-                    p.populate_set_out('creditCard;{}'.format(filename), 'BrowseWarningPaste')
+                    p.populate_set_out('creditcard;{}'.format(filename), 'BrowseWarningPaste')
                 else:
                     publisher.info('{}CreditCard related'.format(to_print))
         else:

bin/Curve.py

@@ -40,9 +40,9 @@ if __name__ == "__main__":
 
     # REDIS #
     r_serv1 = redis.StrictRedis(
-        host=p.config.get("Redis_Level_DB", "host"),
+        host=p.config.get("Redis_Level_DB_Curve", "host"),
-        port=p.config.get("Redis_Level_DB", "port"),
+        port=p.config.get("Redis_Level_DB_Curve", "port"),
-        db=p.config.get("Redis_Level_DB", "db"))
+        db=p.config.get("Redis_Level_DB_Curve", "db"))
 
     # FUNCTIONS #
     publisher.info("Script Curve started")

bin/LAUNCH.sh

@@ -67,6 +67,7 @@ function launching_lvldb {
     db1_y='2013'
     db2_y='2014'
     db3_y='2016'
+    db4_y='3016'
     nb_db=13
 
     screen -dmS "LevelDB"
@@ -78,6 +79,10 @@ function launching_lvldb {
     screen -S "LevelDB" -X screen -t "2014" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2014/ -P '$db2_y' -M '$nb_db'; read x'
     sleep 0.1
     screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db3_y' -M '$nb_db'; read x'
+
+    # For Curve
+    sleep 0.1
+    screen -S "LevelDB" -X screen -t "3016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'3016/ -P '$db4_y' -M '$nb_db'; read x'
 }
 
 function launching_logs {

bin/SQLInjectionDetection.py

@@ -77,7 +77,7 @@ def analyse(url, path):
             #Send to duplicate
             p.populate_set_out(path, 'Duplicate')
             #send to Browse_warning_paste
-            p.populate_set_out('sqlInjectionDetection;{}'.format(path), 'BrowseWarningPaste')
+            p.populate_set_out('sqlinjection;{}'.format(path), 'BrowseWarningPaste')
         else: 
             print "Potential SQL injection:"
             print urllib2.unquote(url)

bin/Url.py

@@ -3,6 +3,7 @@
 import redis
 import pprint
 import time
+import os
 import dns.exception
 from packages import Paste
 from packages import lib_refine
@@ -39,6 +40,10 @@ if __name__ == "__main__":
         port=p.config.getint("Redis_Cache", "port"),
         db=p.config.getint("Redis_Cache", "db"))
 
+    # Protocol file path
+    protocolsfile_path = os.path.join(os.environ['AIL_HOME'],
+                         p.config.get("Directories", "protocolsfile"))
+
     # Country to log as critical
     cc_critical = p.config.get("Url", "cc_critical")
 
@@ -52,7 +57,14 @@ if __name__ == "__main__":
     prec_filename = None
     faup = Faup()
 
-    url_regex = "(http|https|ftp)\://([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.(com|edu|gov|int|mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2}))(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*"
+    # Get all uri from protocolsfile (Used for Curve)
+    uri_scheme = ""
+    with open(protocolsfile_path, 'r') as scheme_file:
+        for scheme in scheme_file:
+            uri_scheme += scheme[:-1]+"|"
+    uri_scheme = uri_scheme[:-1]
+
+    url_regex = "("+uri_scheme+")\://([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.(com|edu|gov|int|mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2}))(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*"
 
     while True:
         if message is not None:

bin/WebStats.py

@@ -26,6 +26,7 @@ def analyse(server, field_name, date, url_parsed):
         prev_score = server.hget(field, date)
         if prev_score is not None:
             server.hset(field, date, int(prev_score) + 1)
+
         else:
             server.hset(field, date, 1)
             if field_name == "domain": #save domain in a set for the monthly plot
@@ -67,7 +68,6 @@ def compute_progression(server, field_name, num_day, url_parsed):
             divisor = value_list_reversed[i-1] if value_list_reversed[i-1] != 0 else 1
             keyword_increase += value_list_reversed[i] / divisor
 
-        
         # filter
         if (keyword_total_sum > threshold_total_sum) and (keyword_increase > threshold_increase):
             

var/www/Flask_server.py

@@ -164,7 +164,6 @@ def showpaste(content_range):
 def getPastebyType(server, module_name):
     all_path = []
     for path in server.smembers('WARNING_'+module_name):
-        #all_path.append("/home/mokaddem/AIL-framework/PASTES/archive/paste.debian.net/2016/06/30/771058.gz")
         all_path.append(path)
     return all_path
 
@@ -198,7 +197,16 @@ def get_top_relevant_data(server, module_name):
             member_set.insert(0, ("passed_days", days))
             return member_set
 
-
+# ========= CACHE CONTROL ========
+@app.after_request
+def add_header(response):
+    """
+    Add headers to both force latest IE rendering engine or Chrome Frame,
+    and also to cache the rendered page for 10 minutes.
+    """
+    response.headers['X-UA-Compatible'] = 'IE=Edge,chrome=1'
+    response.headers['Cache-Control'] = 'public, max-age=0'
+    return response
 
 # ============ ROUTES ============
 

var/www/static/js/plot-graph.js

@@ -112,19 +112,21 @@ function Graph(id_pannel, path, header_size){
 
         // display the top headers
         function setVis(max_display){
-            headings = this.graph.getLabels();
+            var headings = this.graph.getLabels();
             headings.splice(0,1);
             var sorted_list = new Array();
-            today = new Date().getDate()-1; // Take the top from yesterday so that we can see the current evolution
+            var today = new Date().getDate();
+            //today = new Date().getDate()-1; // Could take the top from yesterday so that we can see the current evolution
             for( i=0; i<headings.length; i++){
-                the_heading = headings[i];
+                var the_heading = headings[i];
-                //console.log('heading='+the_heading+' tab['+(today-1)+']['+(parseInt(i)+1)+']='+g.getValue(today-1, parseInt(i)+1));
+                //console.log('heading='+the_heading+' tab['+(today)+']['+(parseInt(i)+1)+']='+this.graph.getValue(today-1, parseInt(i)+1));
                 sorted_list.push({dom: the_heading, val: this.graph.getValue(today-1, parseInt(i)+1), index: parseInt(i)});
             }
             sorted_list.sort(function(a,b) {
                     return b.val - a.val;
             });
 
+
             var display_list = sorted_list.slice(0, max_display);
             for( i=0; i<display_list.length; i++){
                 this.graph.setVisibility(display_list[i].index, true);

var/www/templates/Trending.html

@@ -5,6 +5,10 @@
 	  <meta charset="utf-8">
 	  <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
+          <meta http-equiv="Cache-Control" content="no-cache, no-store, must-revalidate" />
+          <meta http-equiv="Pragma" content="no-cache" />
+          <meta http-equiv="Expires" content="0" />
+
 	  <title>Analysis Information Leak framework Dashboard</title>
 
 	  <!-- Core CSS -->

var/www/templates/browse_important_paste.html

@@ -99,7 +99,7 @@
         <li name='nav-pan'><a data-toggle="tab" href="#creditcard-tab" data-attribute-name="creditcard" data-panel="creditcard-panel">Credit cards</a></li>
         <li name='nav-pan'><a data-toggle="tab" href="#sqlinjection-tab" data-attribute-name="sqlinjection" data-panel="sqlinjection-panel">SQL injections</a></li>
         <li name='nav-pan'><a data-toggle="tab" href="#cve-tab" data-attribute-name="cve" data-panel="cve-panel">CVEs</a></li>
-        <li name='nav-pan'><a data-toggle="tab" href="#key-tab" data-attribute-name="key" data-panel="key-panel">Keys</a></li>
+        <li name='nav-pan'><a data-toggle="tab" href="#keys-tab" data-attribute-name="keys" data-panel="keys-panel">Keys</a></li>
         <li name='nav-pan'><a data-toggle="tab" href="#mail-tab" data-attribute-name="mail" data-panel="mail-panel">Mails</a></li>
         <li name='nav-pan'><a data-toggle="tab" href="#phone-tab" data-attribute-name="phone" data-panel="phone-panel">Phones</a></li>
     </ul>
@@ -119,7 +119,7 @@
         <div class="col-lg-12 tab-pane fade" id="cve-tab">
             <img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" style="margin: 4px;">
         </div>
-        <div class="col-lg-12 tab-pane fade" id="key-tab">
+        <div class="col-lg-12 tab-pane fade" id="keys-tab">
             <img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" style="margin: 4px;">
         </div>
         <div class="col-lg-12 tab-pane fade" id="mail-tab">
@@ -150,7 +150,6 @@ var previous_tab = $('[data-attribute-name="credential');
 var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_for('static', filename='image/loading.gif') }}\" height='26' width='26' style='margin: 4px;'>";
 
 $('.nav-tabs a').on('shown.bs.tab', function(event){
-    console.log(event.target);
     var dataPath = $(event.target).attr('data-attribute-name');
     $.get("{{ url_for('importantPasteByModule') }}"+"?moduleName="+dataPath, function(data, status){
         var currentTab = $('[name].active').children();