Merge branch 'master' into base64

bin/BankAccount.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+"""
+The BankAccount Module
+======================
+
+It apply IBAN regexes on paste content and warn if above a threshold.
+
+"""
+
+import redis
+import time
+import re
+import string
+from itertools import chain
+
+from packages import Paste
+from pubsublogger import publisher
+
+from Helper import Process
+
+import signal
+
+class TimeoutException(Exception):
+    pass
+
+def timeout_handler(signum, frame):
+    raise TimeoutException
+
+signal.signal(signal.SIGALRM, timeout_handler)
+
+_LETTERS_IBAN = chain(enumerate(string.digits + string.ascii_uppercase),
+                 enumerate(string.ascii_lowercase, 10))
+LETTERS_IBAN = {ord(d): str(i) for i, d in _LETTERS_IBAN}
+
+def iban_number(iban):
+    return (iban[4:] + iban[:4]).translate(LETTERS_IBAN)
+
+def is_valid_iban(iban):
+    iban_numb = iban_number(iban)
+    iban_numb_check = iban_number(iban[:2] + '00' + iban[4:])
+    check_digit = '{:0>2}'.format(98 - (int(iban_numb_check) % 97))
+    if check_digit == iban[2:4] and int(iban_numb) % 97 == 1:
+        # valid iban
+        print('valid iban')
+        return True
+    return False
+
+def check_all_iban(l_iban, paste, filename):
+    nb_valid_iban = 0
+    for iban in l_iban:
+        iban = iban[0]+iban[1]+iban[2]
+        iban = ''.join(e for e in iban if e.isalnum())
+        #iban = iban.upper()
+        res = iban_regex_verify.findall(iban)
+        if res:
+            print('checking '+iban)
+            if is_valid_iban(iban):
+                print('------')
+                nb_valid_iban = nb_valid_iban + 1
+
+    if(nb_valid_iban > 0):
+        to_print = 'Iban;{};{};{};'.format(paste.p_source, paste.p_date, paste.p_name)
+        publisher.warning('{}Checked found {} IBAN;{}'.format(
+            to_print, nb_valid_iban, paste.p_path))
+        msg = 'infoleak:automatic-detection="iban";{}'.format(filename)
+        p.populate_set_out(msg, 'Tags')
+
+        #Send to duplicate
+        p.populate_set_out(filename, 'Duplicate')
+
+if __name__ == "__main__":
+    publisher.port = 6380
+    publisher.channel = "Script"
+
+    config_section = 'BankAccount'
+
+    p = Process(config_section)
+    max_execution_time = p.config.getint("BankAccount", "max_execution_time")
+
+    publisher.info("BankAccount started")
+
+    message = p.get_from_set()
+
+    #iban_regex = re.compile(r'\b[A-Za-z]{2}[0-9]{2}(?:[ ]?[0-9]{4}){4}(?:[ ]?[0-9]{1,2})?\b')
+    iban_regex = re.compile(r'\b([A-Za-z]{2}[ \-]?[0-9]{2})(?=(?:[ \-]?[A-Za-z0-9]){9,30})((?:[ \-]?[A-Za-z0-9]{3,5}){2,6})([ \-]?[A-Za-z0-9]{1,3})\b')
+    iban_regex_verify = re.compile(r'^([A-Z]{2})([0-9]{2})([A-Z0-9]{9,30})$')
+
+
+    while True:
+
+        message = p.get_from_set()
+
+        if message is not None:
+
+            filename = message
+            paste = Paste.Paste(filename)
+            content = paste.get_p_content()
+
+            signal.alarm(max_execution_time)
+            try:
+                l_iban = iban_regex.findall(content)
+            except TimeoutException:
+                 print ("{0} processing timeout".format(paste.p_path))
+                 continue
+            else:
+                signal.alarm(0)
+
+            if(len(l_iban) > 0):
+                check_all_iban(l_iban, paste, filename)
+
+        else:
+            publisher.debug("Script BankAccount is Idling 10s")
+            time.sleep(10)

bin/Credential.py

@@ -79,8 +79,6 @@ if __name__ == "__main__":
         content = paste.get_p_content()
         creds = set(re.findall(regex_cred, content))
 
-        publisher.warning('to_print')
-
         if len(creds) == 0:
             continue
 

bin/Helper.py

@@ -146,12 +146,15 @@ class Process(object):
     def populate_set_in(self):
         # monoproc
         src = self.modules.get(self.subscriber_name, 'subscribe')
-        self.pubsub.setup_subscribe(src)
+        if src != 'Redis':
-        for msg in self.pubsub.subscribe():
+            self.pubsub.setup_subscribe(src)
-            in_set = self.subscriber_name + 'in'
+            for msg in self.pubsub.subscribe():
-            self.r_temp.sadd(in_set, msg)
+                in_set = self.subscriber_name + 'in'
-            self.r_temp.hset('queues', self.subscriber_name,
+                self.r_temp.sadd(in_set, msg)
-                             int(self.r_temp.scard(in_set)))
+                self.r_temp.hset('queues', self.subscriber_name,
+                                 int(self.r_temp.scard(in_set)))
+        else:
+            print('{} has no suscriber'.format(self.subscriber_name))
 
     def get_from_set(self):
         # multiproc

bin/Keys.py

@@ -71,6 +71,14 @@ def search_key(paste):
         p.populate_set_out(msg, 'Tags')
         find = True
 
+    if '---- BEGIN SSH2 ENCRYPTED PRIVATE KEY ----' in content:
+        publisher.warning('{} has an ssh2 private key message'.format(paste.p_name))
+        print('SSH2 private key message found')
+
+        msg = 'infoleak:automatic-detection="private-ssh-key";{}'.format(message)
+        p.populate_set_out(msg, 'Tags')
+        find = True
+
     if '-----BEGIN OpenVPN Static key V1-----' in content:
         publisher.warning('{} has an openssh private key message'.format(paste.p_name))
         print('OpenVPN Static key message found')

bin/LAUNCH.sh

@@ -144,6 +144,8 @@ function launching_scripts {
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "CreditCards" bash -c 'cd '${AIL_BIN}'; ./CreditCards.py; read x'
     sleep 0.1
+    screen -S "Script_AIL" -X screen -t "BankAccount" bash -c 'cd '${AIL_BIN}'; ./BankAccount.py; read x'
+    sleep 0.1
     screen -S "Script_AIL" -X screen -t "Onion" bash -c 'cd '${AIL_BIN}'; ./Onion.py; read x'
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "Mail" bash -c 'cd '${AIL_BIN}'; ./Mail.py; read x'

bin/Mixer.py

@@ -68,6 +68,12 @@ if __name__ == '__main__':
         db=cfg.getint("Redis_Mixer_Cache", "db"),
         decode_responses=True)
 
+    server_cache = redis.StrictRedis(
+        host=cfg.get("Redis_Log_submit", "host"),
+        port=cfg.getint("Redis_Log_submit", "port"),
+        db=cfg.getint("Redis_Log_submit", "db"),
+        decode_responses=True)
+
     # LOGGING #
     publisher.info("Feed Script started to receive & publish.")
 
@@ -184,7 +190,17 @@ if __name__ == '__main__':
                 publisher.debug("Empty Paste: {0} not processed".format(message))
         else:
             print("Empty Queues: Waiting...")
+
             if int(time.time() - time_1) > refresh_time:
+                # update internal feeder
+                list_feeder = server_cache.hkeys("mixer_cache:list_feeder")
+                if list_feeder:
+                    for feeder in list_feeder:
+                        count = int(server_cache.hget("mixer_cache:list_feeder", feeder))
+                        if count is None:
+                            count = 0
+                        processed_paste_per_feeder[feeder] = processed_paste_per_feeder.get(feeder, 0) + count
+                        processed_paste = processed_paste + count
                 print(processed_paste_per_feeder)
                 to_print = 'Mixer; ; ; ;mixer_all All_feeders Processed {0} paste(s) in {1}sec'.format(processed_paste, refresh_time)
                 print(to_print)
@@ -204,5 +220,8 @@ if __name__ == '__main__':
                     duplicated_paste_per_feeder[feeder] = 0
 
                 time_1 = time.time()
+
+                # delete internal feeder list
+                server_cache.delete("mixer_cache:list_feeder")
             time.sleep(0.5)
             continue

bin/feeder/pystemon-feeder.py

@@ -62,12 +62,13 @@ while True:
     print(paste)
     if paste is None:
         continue
-    socket.send("%d %s" % (topic, paste))
+    socket.send_string("%d %s" % (topic, paste))
     topic = 102
     try:
-        messagedata = open(pystemonpath+paste).read()
+        with open(pystemonpath+paste, 'rb') as f: #.read()
-        socket.send("%d %s %s" % (topic, paste, base64.b64encode(messagedata)))
+            messagedata = f.read()
-        sleep_inc = sleep_inc-0.01 if sleep_inc-0.01 > 0 else 0
+            socket.send_string("%d %s %s" % (topic, paste, base64.b64encode(messagedata).decode()))
+            sleep_inc = sleep_inc-0.01 if sleep_inc-0.01 > 0 else 0
     except IOError as e:
         # file not found, could be a buffering issue -> increase sleeping time
         print('IOError: Increasing sleep time')

bin/packages/config.cfg.sample

@@ -32,6 +32,8 @@ sender_port = 1337
 
 ##### Flask #####
 [Flask]
+#Number of logs to display in the dashboard
+max_dashboard_logs = 15
 #Maximum number of character to display in the toolip
 max_preview_char = 250
 #Maximum number of character to display in the modal
@@ -44,6 +46,9 @@ minute_processed_paste = 10
 DiffMaxLineLength = 10000
 
 #### Modules ####
+[BankAccount]
+max_execution_time = 60
+
 [Categ]
 #Minimum number of match between the paste and the category file
 matchingThreshold=1

bin/packages/modules.cfg

@@ -51,6 +51,10 @@ publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Re
 subscribe = Redis_CreditCards
 publish = Redis_Duplicate,Redis_ModuleStats,Redis_alertHandler,Redis_Tags
 
+[BankAccount]
+subscribe = Redis_Global
+publish = Redis_Duplicate,Redis_Tags
+
 [Mail]
 subscribe = Redis_Mail
 publish = Redis_Duplicate,Redis_ModuleStats,Redis_alertHandler,Redis_Tags
@@ -130,4 +134,5 @@ subscribe = Redis_Global
 publish = Redis_Duplicate,Redis_alertHandler,Redis_Tags
 
 [submit_paste]
+subscribe = Redis
 publish = Redis_Mixer

bin/submit_paste.py

@@ -40,6 +40,9 @@ def create_paste(uuid, paste_content, ltags, ltagsgalaxies, name):
     relay_message = "{0} {1}".format(save_path, gzip64encoded)
     p.populate_set_out(relay_message, 'Mixer')
 
+    # increase nb of paste by feeder name
+    r_serv_log_submit.hincrby("mixer_cache:list_feeder", "submitted", 1)
+
     # add tags
     add_tags(ltags, ltagsgalaxies, full_path)
 

var/www/modules/Flask_config.py

@@ -144,7 +144,9 @@ bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
 
 UPLOAD_FOLDER = os.path.join(os.environ['AIL_FLASK'], 'submitted')
 
-    # VT
+max_dashboard_logs = int(cfg.get("Flask", "max_dashboard_logs"))
+
+# VT
 try:
     from virusTotalKEYS import vt_key
     if vt_key != '':

var/www/modules/PasteSubmit/templates/PasteSubmit.html

@@ -5,7 +5,8 @@
 	  <meta charset="utf-8">
 	  <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-	  <title>Analysis Information Leak framework Dashboard</title>
+	  <title>Submit Paste - AIL</title>
+		<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
 
 	  <!-- Core CSS -->
 	  <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">

var/www/modules/PasteSubmit/templates/edit_tag_export.html

@@ -5,7 +5,8 @@
 	  <meta charset="utf-8">
 	  <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-	  <title>Analysis Information Leak framework</title>
+	  <title>Tags Export - AIL</title>
+		<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
 
 	  <!-- Core CSS -->
 	  <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">

var/www/modules/PasteSubmit/templates/header_PasteSubmit.html

@@ -1 +1 @@
-<li id='page-PasteSubmit'><a href="{{ url_for('PasteSubmit.PasteSubmit_page') }}"><i class="glyphicon glyphicon-new-window white"></i> PasteSubmit </a></li>
+<li id='page-PasteSubmit'><a href="{{ url_for('PasteSubmit.PasteSubmit_page') }}"><i class="glyphicon glyphicon-new-window white"></i> Submit Paste </a></li>

var/www/modules/PasteSubmit/templates/submiting.html

@@ -5,7 +5,8 @@
 	  <meta charset="utf-8">
 	  <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-	  <title>Analysis Information Leak framework Dashboard</title>
+	  <title>Submit Paste - AIL</title>
+		<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
 
 	  <!-- Core CSS -->
 	  <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">

var/www/modules/Tags/templates/Tags.html

@@ -5,7 +5,8 @@
 	  <meta charset="utf-8">
 	  <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-	  <title>Analysis Information Leak framework Dashboard</title>
+	  <title>Tags - AIL</title>
+		<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
 
 	  <!-- Core CSS -->
 	  <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">

var/www/modules/Tags/templates/edit_galaxy.html

@@ -5,7 +5,8 @@
 	  <meta charset="utf-8">
 	  <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-	  <title>Analysis Information Leak framework Dashboard</title>
+	  <title>Edit Galaxy - AIL</title>
+		<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
 
 	  <!-- Core CSS -->
 	  <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">

var/www/modules/Tags/templates/edit_taxonomie.html

@@ -5,7 +5,8 @@
 	  <meta charset="utf-8">
 	  <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-	  <title>Analysis Information Leak framework Dashboard</title>
+	  <title>Edit Taxonomie - AIL</title>
+		<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
 
 	  <!-- Core CSS -->
 	  <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">

var/www/modules/Tags/templates/galaxies.html

@@ -5,7 +5,8 @@
 	  <meta charset="utf-8">
 	  <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-	  <title>Analysis Information Leak framework Dashboard</title>
+	  <title>Galaxies - AIL</title>
+		<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
 
 	  <!-- Core CSS -->
 	  <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">

var/www/modules/Tags/templates/tag_galaxy_info.html

@@ -5,7 +5,8 @@
 	  <meta charset="utf-8">
 	  <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-	  <title>Analysis Information Leak framework Dashboard</title>
+	  <title>Galaxy Tag Info - AIL</title>
+		<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
 
 	  <!-- Core CSS -->
 	  <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">

var/www/modules/Tags/templates/tagged.html

@@ -5,7 +5,8 @@
   <meta charset="utf-8">
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-  <title>Analysis Information Leak framework Dashboard</title>
+  <title>Tags - AIL</title>
+  <link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
 
   <!-- Core CSS -->
   <link href="/static//css/bootstrap.min.css" rel="stylesheet">

var/www/modules/Tags/templates/taxonomies.html

@@ -5,7 +5,8 @@
 	  <meta charset="utf-8">
 	  <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-	  <title>Analysis Information Leak framework Dashboard</title>
+	  <title>Taxonomies - AIL</title>
+		<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
 
 	  <!-- Core CSS -->
 	  <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">

var/www/modules/browsepastes/templates/browse_important_paste.html

@@ -5,7 +5,8 @@
   <meta charset="utf-8">
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-  <title>Analysis Information Leak framework Dashboard</title>
+  <title>Browse Important Paste - AIL</title>
+  <link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
 
   <!-- Core CSS -->
   <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">

var/www/modules/dashboard/Flask_dashboard.py

@@ -5,10 +5,14 @@
     Flask functions and routes for the dashboard page
 '''
 import json
-
+import os
 import datetime
+import time
 import flask
-from flask import Flask, render_template, jsonify, request, Blueprint
+
+from Date import Date
+
+from flask import Flask, render_template, jsonify, request, Blueprint, url_for
 
 # ============ VARIABLES ============
 import Flask_config
@@ -18,6 +22,8 @@ cfg = Flask_config.cfg
 r_serv = Flask_config.r_serv
 r_serv_log = Flask_config.r_serv_log
 
+max_dashboard_logs = Flask_config.max_dashboard_logs
+
 dashboard = Blueprint('dashboard', __name__, template_folder='templates')
 
 # ============ FUNCTIONS ============
@@ -62,12 +68,87 @@ def get_queues(r):
 
     return newData
 
+def get_date_range(date_from, num_day):
+    date = Date(str(date_from[0:4])+str(date_from[4:6]).zfill(2)+str(date_from[6:8]).zfill(2))
+    date_list = []
+
+    for i in range(0, num_day+1):
+        new_date = date.substract_day(i)
+        date_list.append(new_date[0:4] +'-'+ new_date[4:6] +'-'+ new_date[6:8])
+
+    return date_list
+
+def dashboard_alert(log):
+    # check if we need to display this log
+    if len(log)>50:
+        date = log[1:5]+log[6:8]+log[9:11]
+        utc_str = log[1:20]
+        log = log[46:].split(';')
+        if len(log) == 6:
+            time = datetime_from_utc_to_local(utc_str)
+            path = url_for('showsavedpastes.showsavedpaste',paste=log[5])
+
+            res = {'date': date, 'time': time, 'script': log[0], 'domain': log[1], 'date_paste': log[2],
+                  'paste': log[3], 'message': log[4], 'path': path}
+            return res
+        else:
+            return False
+    else:
+        return False
+
+def datetime_from_utc_to_local(utc_str):
+    utc_datetime = datetime.datetime.strptime(utc_str, '%Y-%m-%d %H:%M:%S')
+    now_timestamp = time.time()
+    offset = datetime.datetime.fromtimestamp(now_timestamp) - datetime.datetime.utcfromtimestamp(now_timestamp)
+    local_time_str = (utc_datetime + offset).strftime('%H:%M:%S')
+    return local_time_str
+
 # ============ ROUTES ============
 
 @dashboard.route("/_logs")
 def logs():
     return flask.Response(event_stream(), mimetype="text/event-stream")
 
+@dashboard.route("/_get_last_logs_json")
+def get_last_logs_json():
+    date = datetime.datetime.now().strftime("%Y%m%d")
+
+    max_day_search = 6
+    day_search = 0
+    warning_found = 0
+    warning_to_found = max_dashboard_logs
+
+    last_logs = []
+
+    date_range = get_date_range(date, max_day_search)
+    while max_day_search != day_search and warning_found != warning_to_found:
+
+        filename_warning_log = 'logs/Script_warn-'+ date_range[day_search] +'.log'
+        filename_log = os.path.join(os.environ['AIL_HOME'], filename_warning_log)
+
+        try:
+            with open(filename_log, 'r') as f:
+                lines = f.read().splitlines()
+                curr_index = -1
+                while warning_found != warning_to_found:
+                    try:
+                        # get lasts warning logs
+                        log_warn = dashboard_alert(lines[curr_index])
+                        if log_warn != False:
+                            last_logs.append(log_warn)
+                            warning_found = warning_found + 1
+                        curr_index = curr_index - 1
+
+                    except IndexError:
+                        # check previous warning log file
+                        day_search = day_search + 1
+                        break
+        except FileNotFoundError:
+            # check previous warning log file
+            day_search = day_search + 1
+
+    return jsonify(list(reversed(last_logs)))
+
 
 @dashboard.route("/_stuff", methods=['GET'])
 def stuff():
@@ -78,7 +159,12 @@ def stuff():
 def index():
     default_minute = cfg.get("Flask", "minute_processed_paste")
     threshold_stucked_module = cfg.getint("Module_ModuleInformation", "threshold_stucked_module")
-    return render_template("index.html", default_minute = default_minute, threshold_stucked_module=threshold_stucked_module)
+    log_select = {10, 25, 50, 100}
+    log_select.add(max_dashboard_logs)
+    log_select = list(log_select)
+    log_select.sort()
+    return render_template("index.html", default_minute = default_minute, threshold_stucked_module=threshold_stucked_module,
+                            log_select=log_select, selected=max_dashboard_logs)
 
 # ========= REGISTRATION =========
 app.register_blueprint(dashboard)

var/www/modules/dashboard/templates/index.html

@@ -6,6 +6,7 @@
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
   <title>Analysis Information Leak framework Dashboard</title>
+  <link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
 
   <!-- Core CSS -->
   <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
@@ -31,6 +32,14 @@
       };
       update_values();
   </script>
+  <style>
+			.tableQueue tbody tr td,
+      .tableQueue tbody tr th,
+      .tableQueue thead tr td,
+      .tableQueue thead tr th{
+				padding: 1px;
+			}
+	</style>
 
 </head>
 <body>
@@ -136,10 +145,13 @@
             <div class="pull-right">
                 <label style="padding-bottom:2px;">
 		<select class="form-control input-sm" id="log_select">
-		<option value="10">10</option>
+      {% for log_selection in log_select %}
-		<option value="25">25</option>
+        {% if log_selection == selected %}
-		<option value="50">50</option>
+          <option value="{{ log_selection }}" selected>{{ log_selection }}</option>
-		<option value="100">100</option>
+        {% else %}
+    		  <option value="{{ log_selection }}">{{ log_selection }}</option>
+        {% endif %}
+      {% endfor %}
 		</select>
                 </label>
 		<input id="checkbox_log_info" type="checkbox" value="info"> INFO
@@ -182,6 +194,66 @@
     <script>
         activePage = "page-index";
         $("#"+activePage).addClass("active");
+
+        var tableBody = document.getElementById('tab_body')
+        $.getJSON('/_get_last_logs_json', function(data) {
+          data.forEach(function (d) {
+            var tr = document.createElement('TR')
+            var time = document.createElement('TD')
+            var chan = document.createElement('TD')
+            var level = document.createElement('TD')
+            var scrpt = document.createElement('TD')
+            var src = document.createElement('TD')
+            var pdate = document.createElement('TD')
+            var nam = document.createElement('TD')
+            var msage = document.createElement('TD')
+            var inspect = document.createElement('TD')
+
+            tr.className = "warning";
+            time.appendChild(document.createTextNode(d.time))
+            chan.appendChild(document.createTextNode('Script'))
+            level.appendChild(document.createTextNode('WARNING'))
+            scrpt.appendChild(document.createTextNode(d.script))
+            src.appendChild(document.createTextNode(d.domain))
+            pdate.appendChild(document.createTextNode(d.date_paste))
+            nam.appendChild(document.createTextNode(d.paste))
+
+            var iconspan = document.createElement('SPAN');
+            var message = d.message.split(" ")
+            if (message[0] == "Detected"){
+                iconspan.className = "glyphicon glyphicon-eye-open";
+            }
+            else if (message[0] == "Checked"){
+                iconspan.className = "glyphicon glyphicon-thumbs-up";
+            }
+            iconspan.innerHTML = "&nbsp;";
+            msage.appendChild(iconspan);
+            msage.appendChild(document.createTextNode(message.join(" ")));
+
+            var action_icon_a = document.createElement("A");
+            action_icon_a.setAttribute("TARGET", "_blank");
+            action_icon_a.setAttribute("HREF", d.path);
+            var action_icon_span = document.createElement('SPAN');
+            action_icon_span.className = "fa fa-search-plus";
+            action_icon_a.appendChild(action_icon_span);
+            inspect.appendChild(action_icon_a)
+            inspect.setAttribute("style", "text-align:center;");
+
+
+            tr.appendChild(time)
+            tr.appendChild(chan);
+            tr.appendChild(level);
+            tr.appendChild(scrpt);
+            tr.appendChild(src);
+            tr.appendChild(pdate);
+            tr.appendChild(nam);
+            tr.appendChild(msage);
+            tr.appendChild(inspect);
+
+            tableBody.appendChild(tr);
+
+		      });
+        });
     </script>
 
     <script type="text/javascript">

var/www/modules/search/templates/search.html

@@ -5,7 +5,8 @@
   <meta charset="utf-8">
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-  <title>Analysis Information Leak framework Dashboard</title>
+  <title>Search - AIL</title>
+  <link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
 
   <!-- Core CSS -->
   <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">

var/www/modules/sentiment/templates/header_sentiment.html

@@ -1,4 +1,4 @@
-<li id='page-sentiment'><a class="dropdown-toggle" data-toggle="dropdown" href="#"><i class="fa fa-heart"></i> Sentiment Analysis
+<li id='page-sentiment'><a class="dropdown-toggle" data-toggle="dropdown" href="{{ url_for('sentiments.sentiment_analysis_trending') }}"><i class="fa fa-heart"></i> Sentiment Analysis
     <span class="caret"></span></a>
     <ul class="dropdown-menu">
       <li><a href="{{ url_for('sentiments.sentiment_analysis_trending') }}"><i class="fa fa-bar-chart-o"> </i> Sentiment trending</a></li>

var/www/modules/sentiment/templates/sentiment_analysis_plot_tool.html

@@ -5,7 +5,8 @@
   <meta charset="utf-8">
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-  <title>Analysis Information Leak framework Dashboard</title>
+  <title>Sentiment Plot Tool - AIL</title>
+  <link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
 
   <!-- Core CSS -->
   <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
@@ -68,7 +69,7 @@
                                        <ul id="providerList2">
                                        </ul>
                                    </div>
-                                </div> 
+                                </div>
                             </div>
                             <!-- right column -->
                             <div class="col-lg-3">
@@ -90,7 +91,7 @@
             <!-- /.panel -->
         </div>
     </div>
-    
+
     <!-- Panel PLOT -->
     <div class="row">
         <div class="col-lg-12">

var/www/modules/sentiment/templates/sentiment_analysis_trending.html

@@ -5,7 +5,8 @@
   <meta charset="utf-8">
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-  <title>Analysis Information Leak framework Dashboard</title>
+  <title>Sentiment Trending - AIL</title>
+  <link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
 
   <!-- Core CSS -->
   <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
@@ -36,7 +37,7 @@
       strong {
           font-size: 16px;
       }
-      
+
       .table {
           margin-bottom: 0px;
       }
@@ -45,7 +46,7 @@
           padding-left:0;
           list-style:none
       }
-      
+
       .sparkLineStats {
           position: relative;
           margin-bottom: -4px;
@@ -64,7 +65,7 @@
       .sparkLineStats ul li div:first-child {
           margin-right: 5px;
       }
-      
+
       .panelInside {
           padding: 5px;
       }
@@ -119,7 +120,7 @@
                                    <div id="today_divr" class="sparkLineStats">
 
                                    </div>
-                                </div> 
+                                </div>
                             </div>
                             <!-- right column -->
                             <div class="col-lg-3">
@@ -167,9 +168,9 @@
                                 </div>
                                 <div class="col-lg-6">
                                    <div id="week_divr" class="sparkLineStats">
-                                       
+
                                    </div>
-                                </div> 
+                                </div>
                             </div>
                             <!-- right column -->
                             <div class="col-lg-3">
@@ -221,7 +222,7 @@
         </div>
     </div>
 
-    
+
     <!-- /.row -->
     </div>
 <!-- /#page-wrapper -->
@@ -236,7 +237,7 @@
     $(document).ready(function(){
         activePage = $('h1.page-header').attr('data-page');
         $("#"+activePage).addClass("active");
-        $('[data-toggle="tooltip"]').tooltip(); 
+        $('[data-toggle="tooltip"]').tooltip();
 
         $("#LoadAll").click(function(){ draw_page("True"); });
         draw_page("False");

var/www/modules/showpaste/Flask_showpaste.py

@@ -8,7 +8,7 @@ import redis
 import json
 import os
 import flask
-from flask import Flask, render_template, jsonify, request, Blueprint, make_response, redirect, url_for
+from flask import Flask, render_template, jsonify, request, Blueprint, make_response, redirect, url_for, Response
 import difflib
 import ssdeep
 
@@ -209,7 +209,7 @@ def showsavedrawpaste():
     requested_path = request.args.get('paste', '')
     paste = Paste.Paste(requested_path)
     content = paste.get_p_content()
-    return content, 200, {'Content-Type': 'text/plain'}
+    return Response(content, mimetype='text/plain')
 
 @showsavedpastes.route("/showpreviewpaste/")
 def showpreviewpaste():

var/www/modules/showpaste/templates/show_saved_paste.html

@@ -1,7 +1,8 @@
 <!DOCTYPE html>
 <html lang="en">
 <head>
-  <title>Paste information</title>
+  <title>Paste information - AIL</title>
+  <link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
   <meta charset="utf-8">
   <meta name="viewport" content="width=device-width, initial-scale=1">
 

var/www/modules/terms/templates/credentials_tracker.html

@@ -4,9 +4,10 @@
     <head>
         <meta charset="utf-8">
         <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    
+
-        <title>Analysis Information Leak framework Dashboard</title>
+        <title>Credentials Tracker - AIL</title>
-    
+        <link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
         <!-- Core CSS -->
         <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
         <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
@@ -24,19 +25,19 @@
         <script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
         <script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
         <script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
-    
+
         <style>
           .btn-link {
               color: #000000
           }
         </style>
-    
+
     </head>
     <body>
         <!-- Modal -->
         <div id="mymodal" class="modal fade" role="dialog">
           <div class="modal-dialog modal-lg">
-        
+
             <!-- Modal content-->
             <div id="mymodalcontent" class="modal-content">
               <div id="mymodalbody" class="modal-body" max-width="8500px">
@@ -51,7 +52,7 @@
           </div>
         </div>
         {% include 'navbar.html' %}
-    
+
         <div id="page-wrapper">
             <div class="row">
               <div class="col-lg-12">
@@ -110,7 +111,7 @@
 
                     $('[data-toggle="tooltip"]').tooltip();
                     table_track = $('#myTable').DataTable({ "order": [[ 1, "dec" ]] });
-            
+
                     table_track.on( 'draw.dt', function () {
                         perform_binding();
                     });
@@ -146,7 +147,7 @@
                         var url = "{{ url_for('terms.credentials_management_query_paste')  }}?cred=" + encodeURIComponent($(this).attr('data-term'));
                         $.ajax({
                             type: 'POST',
-                            url: url, 
+                            url: url,
                             dataType: "json",
                             data: JSON.stringify({ 'allPath': JSON.parse($(this).attr('data-path')) }),
                             contentType : "application/json"
@@ -190,10 +191,10 @@
                         }
                     )});
                 }
-                
+
                 function perform_operation(){
                     var curr_section = $(this).attr('data-section');
-                    var curr_action = $(this).attr('data-action'); 
+                    var curr_action = $(this).attr('data-action');
                     if (curr_action == "add") {
                         var curr_term = $('#'+curr_section+'Input').val();
                     } else if (curr_action == "seek") {
@@ -202,13 +203,13 @@
                         var curr_term = $(this).attr('data-content');
                     }
                     var data_to_send = { section: curr_section, action:curr_action, term: curr_term, extensive: $("#extensive").is(":checked")};
-                
+
                     if (curr_term != "") {
                         //console.log(data_to_send);
                         $.get("{{ url_for('terms.cred_management_action') }}", data_to_send, function(data, status){
                             if(status == "success") {
                                 var json = data;
-                
+
                                 if(json.action == "add") {
                                     //not used for the moment
 
@@ -231,16 +232,16 @@
                                             $( "#nodata" ).text(curr_term);
                                             $( "#nodata" ).fadeIn( "fast");
                                             toAdd = "</button><span data-toggle=\"modal\" data-target=\"#mymodal\" data-term=\""+rep.usr[i]+"\" data-path=\"["+rep.path[i]+"]\" ><button class=\"btn-link\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Show concerned paste(s)\"><span class=\"glyphicon glyphicon-info-sign\"></span></button></span>";
-                                            table_track.row.add( [ 
+                                            table_track.row.add( [
-                                                    rep.usr[i], 
+                                                    rep.usr[i],
-                                                    rep.simil[i], 
+                                                    rep.simil[i],
-                                                    rep.numPaste[i], 
+                                                    rep.numPaste[i],
                                                     toAdd+action_button ] ).draw( false );
                                         }
                                         perform_binding();
                                         perform_modal_binding();
                                     }
-                                }                
+                                }
                             }
                         });
                     }

var/www/modules/terms/templates/header_terms.html

@@ -1,4 +1,4 @@
-<li id='page-termsfrequency'><a class="dropdown-toggle" data-toggle="dropdown" href="#"><i class="fa fa-eye"></i> Terms frequency
+<li id='page-termsfrequency'><a class="dropdown-toggle" data-toggle="dropdown" href="{{ url_for('terms.terms_management') }}"><i class="fa fa-eye"></i> Terms frequency
     <span class="caret"></span></a>
     <ul class="dropdown-menu">
       <li><a href="{{ url_for('terms.terms_management') }}"><i class="fa fa-gear "> </i> Terms managements</a></li>

var/www/modules/terms/templates/terms_management.html

@@ -5,7 +5,8 @@
   <meta charset="utf-8">
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-  <title>Analysis Information Leak framework Dashboard</title>
+  <title>Terms Management</title>
+  <link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
 
   <!-- Core CSS -->
   <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">

var/www/modules/terms/templates/terms_plot_tool.html

@@ -5,7 +5,8 @@
   <meta charset="utf-8">
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-  <title>Analysis Information Leak framework Dashboard</title>
+  <title>Terms Plot Tool - AIL</title>
+  <link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
 
   <!-- Core CSS -->
   <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
@@ -82,7 +83,7 @@
             <!-- /.panel -->
         </div>
     </div>
-    
+
     <!-- Panel PLOT -->
     <div class="row">
         <div class="col-lg-12">
@@ -138,9 +139,9 @@
 
         $( "#amount" ).val( new Date($( ".sliderRange" ).slider( "values", 0 )).toLocaleDateString() +
           " - " + new Date($( ".sliderRange" ).slider( "values", 1 )).toLocaleDateString() );
-        
+
-        $('#plot-btn').click(plotData); 
+        $('#plot-btn').click(plotData);
-        $('#plot-btn-add').click(addData); 
+        $('#plot-btn-add').click(addData);
 
         $("#TermInput").val($("#TermInput").attr("data-init-plot"));
         if($("#TermInput").attr("data-init-plot") != "") {
@@ -163,16 +164,16 @@ var graph_data = [];
 var plotted_terms = [];
 var graph_options = {
                         series: {
-                            lines: { 
+                            lines: {
                                 show: true,
                                 lineWidth: 2
                             },
                             bars: {show: false, barWidth: 60*60*1000},
                             shadowSize: 0
                         },
-                        grid: { 
+                        grid: {
-                            hoverable: true, 
+                            hoverable: true,
-                            clickable: true, 
+                            clickable: true,
                             tickColor: "#f9f9f9",
                             borderWidth: 0
                         },
@@ -203,7 +204,7 @@ function plotData() {
         for(i=1; i<data.length; i++) {
             curr_data.push([data[i][0]*1000, data[i][1]]);
         }
-        to_plot.push({ data: curr_data, label: term}); 
+        to_plot.push({ data: curr_data, label: term});
         graph_data.push({ data: curr_data, label: term});
         plot = $.plot($("#graph"), to_plot, graph_options);
 
@@ -212,7 +213,7 @@ function plotData() {
                 var date = new Date(item.datapoint[0]);
                 var x = parseInt(date.getUTCMonth())+1 + "/" + date.getUTCDate();
                 var y = item.datapoint[1];
-    
+
                 $("#tooltip").html(item.series.label + " for "+x + " = " + y)
                     .css({top: item.pageY-15, left: item.pageX+5})
                     .fadeIn(200);
@@ -264,13 +265,11 @@ function replot() {
             $("#TermInput").val("");
         }))
     }
-    
+
     $.when.apply($, promises).done( function () {
         plot = $.plot($("#graph"), graph_data, graph_options);
     });
-    
+
 }
 
 </script>
-
-

var/www/modules/terms/templates/terms_plot_top.html

@@ -5,7 +5,8 @@
   <meta charset="utf-8">
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-  <title>Analysis Information Leak framework Dashboard</title>
+  <title>Terms Plot Top - AIL</title>
+  <link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
 
   <!-- Core CSS -->
   <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">

var/www/modules/trendingcharts/templates/Trending.html

@@ -9,7 +9,8 @@
           <meta http-equiv="Pragma" content="no-cache" />
           <meta http-equiv="Expires" content="0" />
 
-	  <title>Analysis Information Leak framework Dashboard</title>
+	  <title>Trending Charts - AIL</title>
+		<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
 
 	  <!-- Core CSS -->
 	  <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
@@ -22,7 +23,7 @@
           <script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
           <script src="{{ url_for('static', filename='js/jquery.flot.pie.js') }}"></script>
           <script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
-          <script> 
+          <script>
               var default_display = {{ default_display }};
               var current_displayed_graph;
           </script>
@@ -50,7 +51,7 @@
                 <li><a data-toggle="tab" href="#words-tab" data-pannel="WordTrending" data-path="../static//csv/wordstrendingdata.csv">Words</a></li>
             </ul>
             </br>
-              
+
 <script>
 var chart_1_num_day = 5;
 var chart_2_num_day = 15;
@@ -59,7 +60,7 @@ $SCRIPT_ROOT = {{ request.script_root|tojson|safe }};
 
 <script type="text/javascript" src="{{ url_for('static', filename='js/trendingchart.js')}}"></script>
 
-            <div class="tab-content">  
+            <div class="tab-content">
                 <div class="col-lg-12 tab-pane fade in active" id="tld-tab" >
                      {% include 'trending_graphs/Tldstrending.html' %}
                 </div>
@@ -71,7 +72,7 @@ $SCRIPT_ROOT = {{ request.script_root|tojson|safe }};
                 </div>
                 <div class="col-lg-12 tab-pane fade" id="words-tab">
                      {% include 'trending_graphs/Wordstrending.html' %}
-                </div> 
+                </div>
             </div> <!-- tab-content -->
             <!-- /.row -->
         </div>
@@ -93,7 +94,7 @@ $SCRIPT_ROOT = {{ request.script_root|tojson|safe }};
                     $("[flash-"+attr_name+"]").css('color', '#fece00');
                     setTimeout(function() { $("[flash-"+attr_name+"]").css('color', 'black'); }, 1000);
                     refresh_top_chart(attr_name, false);
-                    if (active_tab_name == attr_name) 
+                    if (active_tab_name == attr_name)
                         plot_top_graph(attr_name, false);
                 }, refresh_interval);
             }
@@ -115,7 +116,7 @@ $SCRIPT_ROOT = {{ request.script_root|tojson|safe }};
 
             // When a pannel is shown, create_and_plot.
             $('.nav-tabs a').on('shown.bs.tab', function(event){
-                create_and_plot($(event.target).attr('data-pannel'), $(event.target).attr('data-path'));    
+                create_and_plot($(event.target).attr('data-pannel'), $(event.target).attr('data-path'));
                 active_tab_name = $(event.target).attr('data-attribute-name')
                 //Top progression chart
                 if(launched_refresher.indexOf($(event.target).attr('data-attribute-name')) == -1){
@@ -133,7 +134,7 @@ $SCRIPT_ROOT = {{ request.script_root|tojson|safe }};
                 $("[align]").css({padding: "2px", width: 'auto', 'background': "rgba(102, 102, 102, 0.15)" , 'border': "3px solid rgb(102, 102, 102)"})
 
                 // Create the graph when the page has just loaded
-                create_and_plot("TldTrending", '../static//csv/tldstrendingdata.csv') 
+                create_and_plot("TldTrending", '../static//csv/tldstrendingdata.csv')
                 //Top progression chart
                 refresh_top_chart("tld", true);
             });

var/www/modules/trendingmodules/templates/Moduletrending.html

@@ -5,7 +5,8 @@
 	  <meta charset="utf-8">
 	  <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-	  <title>Analysis Information Leak framework Dashboard</title>
+	  <title>Modules Statistics - AIL</title>
+		<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
 
 	  <!-- Core CSS -->
 	  <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
@@ -44,7 +45,7 @@
 
                 $("[align]").css({padding: "2px", width: 'auto', 'background': "rgba(102, 102, 102, 0.15)" , 'border': "3px solid rgb(102, 102, 102)"})
 
-               refreshPlot(true); 
+               refreshPlot(true);
             });
 
             function refreshPlot(init){
@@ -63,7 +64,7 @@
                 setTimeout(function() { $("[flash]").css('color', 'black'); }, 1000);
             }
         </script>
-        
+
     </div>
     <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
 </body>

var/www/static/js/indexjavascript.js

@@ -115,7 +115,7 @@ function update_processed_pastes(graph, dataset, graph_type) {
 }
 
 
-// END PROCESSED PASTES    
+// END PROCESSED PASTES
 
 function initfunc( csvay, scroot) {
   window.csv = csvay;
@@ -149,11 +149,11 @@ function create_log_table(obj_json) {
     var chansplit = obj_json.channel.split('.');
     var parsedmess = obj_json.data.split(';');
 
-  
+
     if (parsedmess[0] == "Mixer"){
         var feeder = parsedmess[4].split(" ")[1];
         var paste_processed = parsedmess[4].split(" ")[3];
-        var msg_type = parsedmess[4].split(" ")[2]; 
+        var msg_type = parsedmess[4].split(" ")[2];
 
         if (feeder == "All_feeders"){
             if(list_feeder.indexOf("global") == -1) {
@@ -246,8 +246,9 @@ function create_log_table(obj_json) {
     var action_icon_span = document.createElement('SPAN');
     action_icon_span.className = "fa fa-search-plus";
     action_icon_a.appendChild(action_icon_span);
- 
+
     inspect.appendChild(action_icon_a);
+    inspect.setAttribute("style", "text-align:center;");
 
     tr.appendChild(time)
     tr.appendChild(chan);
@@ -281,7 +282,7 @@ function create_queue_table() {
     document.getElementById("queueing").innerHTML = "";
     var Tablediv = document.getElementById("queueing")
     var table = document.createElement('TABLE')
-    table.className = "table table-bordered table-hover table-striped";
+    table.className = "table table-bordered table-hover table-striped tableQueue";
     var tableHead = document.createElement('THEAD')
     var tableBody = document.createElement('TBODY')
 
@@ -295,7 +296,11 @@ function create_queue_table() {
 
     for (i = 0; i < heading.length; i++) {
         var th = document.createElement('TH')
-        th.width = '100';
+        if (heading[i] == "Amount") {
+          th.width = '50';
+        } else {
+          th.width = '100';
+        }
         th.appendChild(document.createTextNode(heading[i]));
         tr.appendChild(th);
     }
@@ -441,7 +446,7 @@ function load_queues() {
                                 queues_pushed.push(glob_tabvar.row1[i][0].split(".")[0]);
                                 tmp_values.push(parseInt(glob_tabvar.row1[i][1]));
                             }
-                            
+
                         }
                     }
                     tmp_values.unshift(x);
@@ -494,5 +499,3 @@ function manage_undefined() {
 $(document).ready(function () {
     manage_undefined();
 });
-
-