Merge pull request #136 from mokaddem/bugfix

LevelDB auto-directory

bin/BrowseWarningPaste.py

@@ -28,9 +28,11 @@ if __name__ == "__main__":
 
     p = Process(config_section)
 
+    # port generated automatically depending on the date
+    curYear = datetime.now().year
     server = redis.StrictRedis(
                 host=p.config.get("Redis_Level_DB", "host"),
-                port=p.config.get("Redis_Level_DB", "port"),
+                port=curYear,
                 db=p.config.get("Redis_Level_DB", "db"))
 
     # FUNCTIONS #

bin/LAUNCH.sh

@@ -65,33 +65,26 @@ function launching_redis {
 }
 
 function launching_lvldb {
-    #Want to launch more level_db?
     lvdbhost='127.0.0.1'
     lvdbdir="${AIL_HOME}/LEVEL_DB_DATA/"
-    db1_y='2013'
-    db2_y='2014'
-    db3_y='2016'
-    db4_y='2017'
-
-    dbC_y='3016'
     nb_db=13
 
+    db_y=`date +%Y`
+    #Verify that a dir with the correct year exists, create it otherwise
+    if [ ! -d "$lvdbdir$db_y" ]; then
+        mkdir -p "$db_y"
+    fi
+
     screen -dmS "LevelDB"
     sleep 0.1
     echo -e $GREEN"\t* Launching Levels DB servers"$DEFAULT
-    #Add lines here with appropriates options.
-    screen -S "LevelDB" -X screen -t "2013" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2013/ -P '$db1_y' -M '$nb_db'; read x'
-    sleep 0.1
-    screen -S "LevelDB" -X screen -t "2014" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2014/ -P '$db2_y' -M '$nb_db'; read x'
-    sleep 0.1
-    screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db3_y' -M '$nb_db'; read x'
-    sleep 0.1
-    screen -S "LevelDB" -X screen -t "2017" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2017/ -P '$db4_y' -M '$nb_db'; read x'
 
-
+    #Launch a DB for each dir
-    # For Curve
+    for pathDir in $lvdbdir*/ ; do
-    sleep 0.1
+        yDir=$(basename "$pathDir")
-    screen -S "LevelDB" -X screen -t "3016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'3016/ -P '$dbC_y' -M '$nb_db'; read x'
+        sleep 0.1
+        screen -S "LevelDB" -X screen -t "$yDir" bash -c 'redis-leveldb -H '$lvdbhost' -D '$pathDir'/ -P '$yDir' -M '$nb_db'; read x'
+    done
 }
 
 function launching_logs {
@@ -255,6 +248,7 @@ for i in ${!options[@]}; do
             Killall)
                 if [[ $isredis || $islvldb || $islogged || $isqueued || $isscripted ]]; then
                     kill $isredis $islvldb $islogged $isqueued $isscripted
+                    sleep 0.2
                     echo -e $ROSE`screen -ls`$DEFAULT
                     echo -e $GREEN"\t* $isredis $islvldb $islogged $isqueued $isscripted killed."$DEFAULT
                 else

bin/Repartition_graph.py

@@ -4,6 +4,7 @@
 import redis
 import argparse
 import ConfigParser
+from datetime import datetime
 from pubsublogger import publisher
 
 import matplotlib.pyplot as plt
@@ -24,13 +25,16 @@ def main():
     parser.add_argument('-f', type=str, metavar="filename", default="figure",
                         help='The absolute path name of the "figure.png"',
                         action='store')
+    parser.add_argument('-y', '--year', type=int, required=False, default=None, help='The date related to the DB')
 
     args = parser.parse_args()
 
     # REDIS #
+    # port generated automatically depending on the date
+    curYear = datetime.now().year if args.year is None else args.year
     r_serv = redis.StrictRedis(
         host=cfg.get("Redis_Level_DB_Hashs", "host"),
-        port=cfg.getint("Redis_Level_DB_Hashs", "port"),
+        port=curYear,
         db=cfg.getint("Redis_Level_DB_Hashs", "db"))
 
     # LOGGING #

bin/launch_logs.sh

@@ -3,6 +3,10 @@
 set -e
 set -x
 
+[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_LEVELDB" ] && echo "Needs the env var AIL_LEVELDB. Run the script from the virtual environment." && exit 1;
+
 screen -dmS "Logging"
 sleep 0.1
 echo -e $GREEN"\t* Launching logging process"$DEFAULT

bin/launch_lvldb.sh

@@ -3,21 +3,27 @@
 set -e
 set -x
 
+[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_LEVELDB" ] && echo "Needs the env var AIL_LEVELDB. Run the script from the virtual environment." && exit 1;
+
 lvdbhost='127.0.0.1'
 lvdbdir="${AIL_HOME}/LEVEL_DB_DATA/"
-db1_y='2013'
-db2_y='2014'
-db3_y='2016'
-db4_y='3016'
 nb_db=13
 
+db_y=`date +%Y`
+#Verify that a dir with the correct year exists, create it otherwise
+if [ ! -d "$lvdbdir$db_y" ]; then
+    mkdir -p "$db_y"
+fi
+
 screen -dmS "LevelDB"
 sleep 0.1
 echo -e $GREEN"\t* Launching Levels DB servers"$DEFAULT
-sleep 0.1
-screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db3_y' -M '$nb_db'; read x'
-
-# For Curve
-sleep 0.1
-screen -S "LevelDB" -X screen -t "3016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'3016/ -P '$db4_y' -M '$nb_db'; read x'
 
+#Launch a DB for each dir
+for pathDir in $lvdbdir*/ ; do
+    yDir=$(basename "$pathDir")
+    sleep 0.1
+    screen -S "LevelDB" -X screen -t "$yDir" bash -c 'redis-leveldb -H '$lvdbhost' -D '$pathDir'/ -P '$yDir' -M '$nb_db'; read x'
+done

bin/launch_queues.sh

@@ -3,6 +3,10 @@
 set -e
 set -x
 
+[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_LEVELDB" ] && echo "Needs the env var AIL_LEVELDB. Run the script from the virtual environment." && exit 1;
+
 screen -dmS "Queue"
 sleep 0.1
 

bin/launch_redis.sh

@@ -3,6 +3,10 @@
 set -e
 set -x
 
+[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_LEVELDB" ] && echo "Needs the env var AIL_LEVELDB. Run the script from the virtual environment." && exit 1;
+
 conf_dir="${AIL_HOME}/configs/"
 
 screen -dmS "Redis"

bin/launch_scripts.sh

@@ -3,57 +3,75 @@
 set -e
 set -x
 
+[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_LEVELDB" ] && echo "Needs the env var AIL_LEVELDB. Run the script from the virtual environment." && exit 1;
+
+echo -e "\t* Checking configuration"
+bash -c "./Update-conf.py"
+exitStatus=$?
+if [ $exitStatus -ge 1 ]; then
+    echo -e $RED"\t* Configuration not up-to-date"$DEFAULT
+    exit
+fi
+echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT
+
 screen -dmS "Script"
 sleep 0.1
-
 echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
 
-    screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModuleInformation.py -k 0 -c 1; read x'
+screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModulesInformationV2.py -k 0 -c 1; read x'
-    sleep 0.1
+sleep 0.1
-    screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x'
+screen -S "Script" -X screen -t "Mixer" bash -c './Mixer.py; read x'
-    sleep 0.1
+sleep 0.1
-    screen -S "Script" -X screen -t "Duplicates" bash -c './Duplicates.py; read x'
+screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x'
-    sleep 0.1
+sleep 0.1
-    screen -S "Script" -X screen -t "Attributes" bash -c './Attributes.py; read x'
+screen -S "Script" -X screen -t "Duplicates" bash -c './Duplicates.py; read x'
-    sleep 0.1
+sleep 0.1
-    screen -S "Script" -X screen -t "Lines" bash -c './Lines.py; read x'
+screen -S "Script" -X screen -t "Attributes" bash -c './Attributes.py; read x'
-    sleep 0.1
+sleep 0.1
-    screen -S "Script" -X screen -t "DomClassifier" bash -c './DomClassifier.py; read x'
+screen -S "Script" -X screen -t "Lines" bash -c './Lines.py; read x'
-    sleep 0.1
+sleep 0.1
-    screen -S "Script" -X screen -t "Categ" bash -c './Categ.py; read x'
+screen -S "Script" -X screen -t "DomClassifier" bash -c './DomClassifier.py; read x'
-    sleep 0.1
+sleep 0.1
-    screen -S "Script" -X screen -t "Tokenize" bash -c './Tokenize.py; read x'
+screen -S "Script" -X screen -t "Categ" bash -c './Categ.py; read x'
-    sleep 0.1
+sleep 0.1
-    screen -S "Script" -X screen -t "CreditCards" bash -c './CreditCards.py; read x'
+screen -S "Script" -X screen -t "Tokenize" bash -c './Tokenize.py; read x'
-    sleep 0.1
+sleep 0.1
-    screen -S "Script" -X screen -t "Onion" bash -c './Onion.py; read x'
+screen -S "Script" -X screen -t "CreditCards" bash -c './CreditCards.py; read x'
-    sleep 0.1
+sleep 0.1
-    screen -S "Script" -X screen -t "Mail" bash -c './Mail.py; read x'
+screen -S "Script" -X screen -t "Onion" bash -c './Onion.py; read x'
-    sleep 0.1
+sleep 0.1
-    screen -S "Script" -X screen -t "Web" bash -c './Web.py; read x'
+screen -S "Script" -X screen -t "Mail" bash -c './Mail.py; read x'
-    sleep 0.1
+sleep 0.1
-    screen -S "Script" -X screen -t "Credential" bash -c './Credential.py; read x'
+screen -S "Script" -X screen -t "Web" bash -c './Web.py; read x'
-    sleep 0.1
+sleep 0.1
-    screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x'
+screen -S "Script" -X screen -t "Credential" bash -c './Credential.py; read x'
-    sleep 0.1
+sleep 0.1
-    screen -S "Script" -X screen -t "CurveManageTopSets" bash -c './CurveManageTopSets.py; read x'
+screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x'
-    sleep 0.1
+sleep 0.1
-    screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x'
+screen -S "Script" -X screen -t "CurveManageTopSets" bash -c './CurveManageTopSets.py; read x'
-    sleep 0.1
+sleep 0.1
-    screen -S "Script" -X screen -t "Keys" bash -c './Keys.py; read x'
+screen -S "Script" -X screen -t "RegexForTermsFrequency" bash -c './RegexForTermsFrequency.py; read x'
-    sleep 0.1
+sleep 0.1
-    screen -S "Script" -X screen -t "Phone" bash -c './Phone.py; read x'
+screen -S "Script" -X screen -t "SetForTermsFrequency" bash -c './SetForTermsFrequency.py; read x'
-    sleep 0.1
+sleep 0.1
-    screen -S "Script" -X screen -t "Release" bash -c './Release.py; read x'
+screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x'
-    sleep 0.1
+sleep 0.1
-    screen -S "Script" -X screen -t "Cve" bash -c './Cve.py; read x'
+screen -S "Script" -X screen -t "Keys" bash -c './Keys.py; read x'
-    sleep 0.1
+sleep 0.1
-    screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x'
+screen -S "Script" -X screen -t "Phone" bash -c './Phone.py; read x'
-    sleep 0.1
+sleep 0.1
-    screen -S "Script" -X screen -t "ModuleStats" bash -c './ModuleStats.py; read x'
+screen -S "Script" -X screen -t "Release" bash -c './Release.py; read x'
-    sleep 0.1
+sleep 0.1
-    screen -S "Script" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x'
+screen -S "Script" -X screen -t "Cve" bash -c './Cve.py; read x'
-    sleep 0.1
+sleep 0.1
-    screen -S "Script" -X screen -t "Browse_warning_paste" bash -c './Browse_warning_paste.py; read x'
+screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x'
-    sleep 0.1
+sleep 0.1
-    screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x'
+screen -S "Script" -X screen -t "ModuleStats" bash -c './ModuleStats.py; read x'
+sleep 0.1
+screen -S "Script" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x'
+sleep 0.1
+screen -S "Script" -X screen -t "BrowseWarningPaste" bash -c './BrowseWarningPaste.py; read x'
+sleep 0.1
+screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x'

bin/packages/config.cfg.sample

@@ -104,7 +104,6 @@ db = 5
 
 [Redis_Level_DB]
 host = localhost
-port = 2016
 db = 0
 
 [Redis_Level_DB_Trending]
@@ -114,7 +113,6 @@ db = 3
 
 [Redis_Level_DB_Hashs]
 host = localhost
-port = 2016
 db = 1
 
 [Url]

installing_deps.sh

@@ -84,9 +84,9 @@ if [ -z "$VIRTUAL_ENV" ]; then
 
 fi
 
+year1=20`date +%y`
 mkdir -p $AIL_HOME/{PASTES,Blooms,dumps}
-mkdir -p $AIL_HOME/LEVEL_DB_DATA/2016
+mkdir -p $AIL_HOME/LEVEL_DB_DATA/$year1
-mkdir -p $AIL_HOME/LEVEL_DB_DATA/3016
 
 pip install -U pip
 pip install -U -r pip_packages_requirement.txt

var/www/modules/Flask_config.py

@@ -38,11 +38,6 @@ r_serv_charts = redis.StrictRedis(
     port=cfg.getint("Redis_Level_DB_Trending", "port"),
     db=cfg.getint("Redis_Level_DB_Trending", "db"))
 
-r_serv_db = redis.StrictRedis(
-    host=cfg.get("Redis_Level_DB", "host"),
-    port=cfg.getint("Redis_Level_DB", "port"),
-    db=cfg.getint("Redis_Level_DB", "db"))
-
 r_serv_sentiment = redis.StrictRedis(
         host=cfg.get("Redis_Level_DB_Sentiment", "host"),
         port=cfg.getint("Redis_Level_DB_Sentiment", "port"),

var/www/modules/browsepastes/Flask_browsepastes.py

@@ -7,6 +7,8 @@
 import redis
 import json
 import flask
+import os
+from datetime import datetime
 from flask import Flask, render_template, jsonify, request, Blueprint
 
 import Paste
@@ -18,7 +20,25 @@ app = Flask_config.app
 cfg = Flask_config.cfg
 max_preview_char = Flask_config.max_preview_char
 max_preview_modal = Flask_config.max_preview_modal
-r_serv_db = Flask_config.r_serv_db
+
+#init all lvlDB servers
+curYear = datetime.now().year
+r_serv_db = {}
+# port generated automatically depending on available levelDB date
+yearList = []
+lvdbdir= os.path.join(os.environ['AIL_HOME'], "LEVEL_DB_DATA/")
+for year in os.listdir(lvdbdir):
+    try:
+        intYear = int(year)
+    except:
+        continue
+
+    yearList.append([year, intYear, int(curYear) == intYear])
+    r_serv_db[intYear] = redis.StrictRedis(
+        host=cfg.get("Redis_Level_DB", "host"),
+        port=intYear,
+        db=cfg.getint("Redis_Level_DB", "db"))
+yearList.sort(reverse=True)
 
 browsepastes = Blueprint('browsepastes', __name__, template_folder='templates')
 
@@ -31,9 +51,9 @@ def getPastebyType(server, module_name):
     return all_path
 
 
-def event_stream_getImportantPasteByModule(module_name):
+def event_stream_getImportantPasteByModule(module_name, year):
     index = 0
-    all_pastes_list = getPastebyType(r_serv_db, module_name)
+    all_pastes_list = getPastebyType(r_serv_db[year], module_name)
     for path in all_pastes_list:
         index += 1
         paste = Paste.Paste(path)
@@ -57,18 +77,19 @@ def event_stream_getImportantPasteByModule(module_name):
 @browsepastes.route("/browseImportantPaste/", methods=['GET'])
 def browseImportantPaste():
     module_name = request.args.get('moduleName')
-    return render_template("browse_important_paste.html")
+    return render_template("browse_important_paste.html", year_list=yearList, selected_year=curYear)
 
 
 @browsepastes.route("/importantPasteByModule/", methods=['GET'])
 def importantPasteByModule():
     module_name = request.args.get('moduleName')
+    currentSelectYear = int(request.args.get('year'))
 
     all_content = []
     paste_date = []
     paste_linenum = []
     all_path = []
-    allPastes = getPastebyType(r_serv_db, module_name)
+    allPastes = getPastebyType(r_serv_db[currentSelectYear], module_name)
 
     for path in allPastes[0:10]:
         all_path.append(path)
@@ -88,6 +109,7 @@ def importantPasteByModule():
 
     return render_template("important_paste_by_module.html",
             moduleName=module_name, 
+            year=currentSelectYear,
             all_path=all_path, 
             content=all_content, 
             paste_date=paste_date, 
@@ -95,10 +117,11 @@ def importantPasteByModule():
             char_to_display=max_preview_modal, 
             finished=finished)
 
-@browsepastes.route("/_getImportantPasteByModule")
+@browsepastes.route("/_getImportantPasteByModule", methods=['GET'])
 def getImportantPasteByModule():
     module_name = request.args.get('moduleName')
-    return flask.Response(event_stream_getImportantPasteByModule(module_name), mimetype="text/event-stream")
+    currentSelectYear = int(request.args.get('year'))
+    return flask.Response(event_stream_getImportantPasteByModule(module_name, currentSelectYear), mimetype="text/event-stream")
 
 
 # ========= REGISTRATION =========

var/www/modules/browsepastes/templates/browse_important_paste.html

@@ -63,6 +63,20 @@
       </div>
       <!-- /.col-lg-12 -->
   </div>
+
+  <div class="row">
+  <div class="col-md-12" style="margin-bottom: 0.2cm;">
+      <strong style="">Year: </strong> 
+      <select class="form-control" id="index_year" style="display: inline-block; margin-bottom: 5px; width: 5%">
+          {% for yearElem in year_list %}
+          <option {% if yearElem[2] %} selected="selected" {% endif %}  value="{{ yearElem[0] }}" >{{ yearElem[1] }}</option>
+          {% endfor %}
+      </select>
+  </div>
+  </br>
+  </div>
+
+
     <!-- /.row -->
     <div class="row">
 
@@ -117,9 +131,10 @@
         $("#"+activePage).addClass("active");
 
         var dataPath = 'credential';
-        $.get("{{ url_for('browsepastes.importantPasteByModule') }}"+"?moduleName="+dataPath, function(data, status){
+        $.get("{{ url_for('browsepastes.importantPasteByModule') }}"+"?moduleName="+dataPath+"&year="+currentSelectYear, function(data, status){
             $('#'+dataPath+'-tab').html(data);
         });
+
     });
 </script>
 
@@ -127,12 +142,23 @@
 <script>
 // When a pannel is shown, create the data-table.
 var previous_tab = $('[data-attribute-name="credential');
+var currentTabName = previous_tab.attr('data-attribute-name');
 var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_for('static', filename='image/loading.gif') }}\" height='26' width='26' style='margin: 4px;'>";
+var currentSelectYear = {{ selected_year }};
+
+$('#index_year').on('change', function() {
+    currentSelectYear = this.value;
+
+    $.get("{{ url_for('browsepastes.importantPasteByModule') }}"+"?moduleName="+currentTabName+"&year="+currentSelectYear, function(data, status){
+        $('#'+currentTabName+'-tab').html(data);
+    });
+})
 
 $('.nav-tabs a').on('shown.bs.tab', function(event){
     var dataPath = $(event.target).attr('data-attribute-name');
-    $.get("{{ url_for('browsepastes.importantPasteByModule') }}"+"?moduleName="+dataPath, function(data, status){
+    currentTabName = dataPath;
-        var currentTab = $('[name].active').children();
+    $.get("{{ url_for('browsepastes.importantPasteByModule') }}"+"?moduleName="+currentTabName+"&year="+currentSelectYear, function(data, status){
+        currentTab = $('[name].active').children();
         $('#'+previous_tab.attr('data-attribute-name')+'-tab').html(loading_gif);
         currentTab.removeClass( "active" );
 

var/www/modules/browsepastes/templates/important_paste_by_module.html

@@ -45,7 +45,7 @@ function deploy_source() {
     if(typeof(EventSource) !== "undefined" && typeof(source) !== "") {
         $("#load_more_json_button1").show();
         $("#load_more_json_button2").show();
-        var source = new EventSource("{{ url_for('browsepastes.getImportantPasteByModule') }}"+"?moduleName="+moduleName);
+        var source = new EventSource("{{ url_for('browsepastes.getImportantPasteByModule') }}"+"?moduleName="+moduleName+"&year="+currentSelectYear);
         source.onmessage = function(event) {
             var feed = jQuery.parseJSON( event.data );
             curr_numElem = parseInt($("#myTable_"+moduleName).attr('data-numElem'));
@@ -113,6 +113,7 @@ function add_entries_X(to_add) {
 
 <script>
 var moduleName = "{{ moduleName }}";
+var currentSelectYear = "{{ year }}";
 var search_table;
 var last_clicked_paste;
 var can_change_modal_content = true;