diff --git a/bin/BrowseWarningPaste.py b/bin/BrowseWarningPaste.py index 01704902..4f49f56b 100755 --- a/bin/BrowseWarningPaste.py +++ b/bin/BrowseWarningPaste.py @@ -28,9 +28,11 @@ if __name__ == "__main__": p = Process(config_section) + # port generated automatically depending on the date + curYear = datetime.now().year server = redis.StrictRedis( host=p.config.get("Redis_Level_DB", "host"), - port=p.config.get("Redis_Level_DB", "port"), + port=curYear, db=p.config.get("Redis_Level_DB", "db")) # FUNCTIONS # diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index f33af903..227d3a9b 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -65,37 +65,26 @@ function launching_redis { } function launching_lvldb { - #Want to launch more level_db? - #FIXME update the date in config.cfg lvdbhost='127.0.0.1' lvdbdir="${AIL_HOME}/LEVEL_DB_DATA/" - db1_y='2016' - db2_y='2017' - dbn_y=`date +%Y` - - dbC1_y='3016' - dbCn_y=30`date +%y` nb_db=13 + db_y=`date +%Y` + #Verify that a dir with the correct year exists, create it otherwise + if [ ! -d "$lvdbdir$db_y" ]; then + mkdir -p "$db_y" + fi + screen -dmS "LevelDB" sleep 0.1 echo -e $GREEN"\t* Launching Levels DB servers"$DEFAULT - #Add lines here with appropriates options. - sleep 0.1 - screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db1_y' -M '$nb_db'; read x' - sleep 0.1 - screen -S "LevelDB" -X screen -t "2017" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2017/ -P '$db2_y' -M '$nb_db'; read x' - sleep 0.1 - screen -S "LevelDB" -X screen -t "$dbn_y" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir$dbn_y'/ -P '$dbn_y' -M '$nb_db'; read x' - - # For Curve - sleep 0.1 - screen -S "LevelDB" -X screen -t "3016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'3016/ -P '$dbC1_y' -M '$nb_db'; read x' - sleep 0.1 - screen -S "LevelDB" -X screen -t "3017" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'3017/ -P '$dbC1_y' -M '$nb_db'; read x' - sleep 0.1 - screen -S "LevelDB" -X screen -t "$dbCn_y" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir$dbCn_y'/ -P '$dbCn_y' -M '$nb_db'; read x' + #Launch a DB for each dir + for pathDir in $lvdbdir*/ ; do + yDir=$(basename "$pathDir") + sleep 0.1 + screen -S "LevelDB" -X screen -t "$yDir" bash -c 'redis-leveldb -H '$lvdbhost' -D '$pathDir'/ -P '$yDir' -M '$nb_db'; read x' + done } function launching_logs { diff --git a/bin/Repartition_graph.py b/bin/Repartition_graph.py index f1955a0e..38c5e1b6 100755 --- a/bin/Repartition_graph.py +++ b/bin/Repartition_graph.py @@ -4,6 +4,7 @@ import redis import argparse import ConfigParser +from datetime import datetime from pubsublogger import publisher import matplotlib.pyplot as plt @@ -24,13 +25,16 @@ def main(): parser.add_argument('-f', type=str, metavar="filename", default="figure", help='The absolute path name of the "figure.png"', action='store') + parser.add_argument('-y', '--year', type=int, required=False, default=None, help='The date related to the DB') args = parser.parse_args() # REDIS # + # port generated automatically depending on the date + curYear = datetime.now().year if args.year is None else args.year r_serv = redis.StrictRedis( host=cfg.get("Redis_Level_DB_Hashs", "host"), - port=cfg.getint("Redis_Level_DB_Hashs", "port"), + port=curYear, db=cfg.getint("Redis_Level_DB_Hashs", "db")) # LOGGING # diff --git a/bin/launch_logs.sh b/bin/launch_logs.sh index 107a2450..ed08ea48 100755 --- a/bin/launch_logs.sh +++ b/bin/launch_logs.sh @@ -3,6 +3,10 @@ set -e set -x +[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_LEVELDB" ] && echo "Needs the env var AIL_LEVELDB. Run the script from the virtual environment." && exit 1; + screen -dmS "Logging" sleep 0.1 echo -e $GREEN"\t* Launching logging process"$DEFAULT diff --git a/bin/launch_lvldb.sh b/bin/launch_lvldb.sh index ef4dcb2c..a534e7ae 100755 --- a/bin/launch_lvldb.sh +++ b/bin/launch_lvldb.sh @@ -3,21 +3,27 @@ set -e set -x +[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_LEVELDB" ] && echo "Needs the env var AIL_LEVELDB. Run the script from the virtual environment." && exit 1; + lvdbhost='127.0.0.1' lvdbdir="${AIL_HOME}/LEVEL_DB_DATA/" -db1_y='2013' -db2_y='2014' -db3_y='2016' -db4_y='3016' nb_db=13 +db_y=`date +%Y` +#Verify that a dir with the correct year exists, create it otherwise +if [ ! -d "$lvdbdir$db_y" ]; then + mkdir -p "$db_y" +fi + screen -dmS "LevelDB" sleep 0.1 echo -e $GREEN"\t* Launching Levels DB servers"$DEFAULT -sleep 0.1 -screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db3_y' -M '$nb_db'; read x' - -# For Curve -sleep 0.1 -screen -S "LevelDB" -X screen -t "3016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'3016/ -P '$db4_y' -M '$nb_db'; read x' +#Launch a DB for each dir +for pathDir in $lvdbdir*/ ; do + yDir=$(basename "$pathDir") + sleep 0.1 + screen -S "LevelDB" -X screen -t "$yDir" bash -c 'redis-leveldb -H '$lvdbhost' -D '$pathDir'/ -P '$yDir' -M '$nb_db'; read x' +done diff --git a/bin/launch_queues.sh b/bin/launch_queues.sh index 8c2699ea..2ddd2b7d 100755 --- a/bin/launch_queues.sh +++ b/bin/launch_queues.sh @@ -3,6 +3,10 @@ set -e set -x +[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_LEVELDB" ] && echo "Needs the env var AIL_LEVELDB. Run the script from the virtual environment." && exit 1; + screen -dmS "Queue" sleep 0.1 diff --git a/bin/launch_redis.sh b/bin/launch_redis.sh index c7af27c8..69910927 100755 --- a/bin/launch_redis.sh +++ b/bin/launch_redis.sh @@ -3,6 +3,10 @@ set -e set -x +[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_LEVELDB" ] && echo "Needs the env var AIL_LEVELDB. Run the script from the virtual environment." && exit 1; + conf_dir="${AIL_HOME}/configs/" screen -dmS "Redis" diff --git a/bin/launch_scripts.sh b/bin/launch_scripts.sh index 694f0138..e593b11e 100755 --- a/bin/launch_scripts.sh +++ b/bin/launch_scripts.sh @@ -3,57 +3,75 @@ set -e set -x +[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_LEVELDB" ] && echo "Needs the env var AIL_LEVELDB. Run the script from the virtual environment." && exit 1; + +echo -e "\t* Checking configuration" +bash -c "./Update-conf.py" +exitStatus=$? +if [ $exitStatus -ge 1 ]; then + echo -e $RED"\t* Configuration not up-to-date"$DEFAULT + exit +fi +echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT + screen -dmS "Script" sleep 0.1 - echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT - screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModuleInformation.py -k 0 -c 1; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Duplicates" bash -c './Duplicates.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Attributes" bash -c './Attributes.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Lines" bash -c './Lines.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "DomClassifier" bash -c './DomClassifier.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Categ" bash -c './Categ.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Tokenize" bash -c './Tokenize.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "CreditCards" bash -c './CreditCards.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Onion" bash -c './Onion.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Mail" bash -c './Mail.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Web" bash -c './Web.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Credential" bash -c './Credential.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "CurveManageTopSets" bash -c './CurveManageTopSets.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Keys" bash -c './Keys.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Phone" bash -c './Phone.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Release" bash -c './Release.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Cve" bash -c './Cve.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "ModuleStats" bash -c './ModuleStats.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Browse_warning_paste" bash -c './Browse_warning_paste.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x' +screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModulesInformationV2.py -k 0 -c 1; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Mixer" bash -c './Mixer.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Duplicates" bash -c './Duplicates.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Attributes" bash -c './Attributes.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Lines" bash -c './Lines.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "DomClassifier" bash -c './DomClassifier.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Categ" bash -c './Categ.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Tokenize" bash -c './Tokenize.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "CreditCards" bash -c './CreditCards.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Onion" bash -c './Onion.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Mail" bash -c './Mail.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Web" bash -c './Web.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Credential" bash -c './Credential.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "CurveManageTopSets" bash -c './CurveManageTopSets.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "RegexForTermsFrequency" bash -c './RegexForTermsFrequency.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "SetForTermsFrequency" bash -c './SetForTermsFrequency.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Keys" bash -c './Keys.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Phone" bash -c './Phone.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Release" bash -c './Release.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Cve" bash -c './Cve.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "ModuleStats" bash -c './ModuleStats.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "BrowseWarningPaste" bash -c './BrowseWarningPaste.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x' diff --git a/bin/packages/config.cfg.sample b/bin/packages/config.cfg.sample index 2e2d246d..1adaf04d 100644 --- a/bin/packages/config.cfg.sample +++ b/bin/packages/config.cfg.sample @@ -104,7 +104,6 @@ db = 5 [Redis_Level_DB] host = localhost -port = 2016 db = 0 [Redis_Level_DB_Trending] @@ -114,7 +113,6 @@ db = 3 [Redis_Level_DB_Hashs] host = localhost -port = 2016 db = 1 [Url] diff --git a/installing_deps.sh b/installing_deps.sh index 9f858810..9c414fe7 100755 --- a/installing_deps.sh +++ b/installing_deps.sh @@ -85,12 +85,8 @@ if [ -z "$VIRTUAL_ENV" ]; then fi year1=20`date +%y` -year2=30`date +%y` mkdir -p $AIL_HOME/{PASTES,Blooms,dumps} mkdir -p $AIL_HOME/LEVEL_DB_DATA/$year1 -mkdir -p $AIL_HOME/LEVEL_DB_DATA/$year2 -mkdir -p $AIL_HOME/LEVEL_DB_DATA/2016 -mkdir -p $AIL_HOME/LEVEL_DB_DATA/3016 pip install -U pip pip install -U -r pip_packages_requirement.txt diff --git a/var/www/modules/Flask_config.py b/var/www/modules/Flask_config.py index 161e6130..5e51c4ac 100644 --- a/var/www/modules/Flask_config.py +++ b/var/www/modules/Flask_config.py @@ -38,11 +38,6 @@ r_serv_charts = redis.StrictRedis( port=cfg.getint("Redis_Level_DB_Trending", "port"), db=cfg.getint("Redis_Level_DB_Trending", "db")) -r_serv_db = redis.StrictRedis( - host=cfg.get("Redis_Level_DB", "host"), - port=cfg.getint("Redis_Level_DB", "port"), - db=cfg.getint("Redis_Level_DB", "db")) - r_serv_sentiment = redis.StrictRedis( host=cfg.get("Redis_Level_DB_Sentiment", "host"), port=cfg.getint("Redis_Level_DB_Sentiment", "port"), diff --git a/var/www/modules/browsepastes/Flask_browsepastes.py b/var/www/modules/browsepastes/Flask_browsepastes.py index d3b1bb4d..30853c38 100644 --- a/var/www/modules/browsepastes/Flask_browsepastes.py +++ b/var/www/modules/browsepastes/Flask_browsepastes.py @@ -7,6 +7,8 @@ import redis import json import flask +import os +from datetime import datetime from flask import Flask, render_template, jsonify, request, Blueprint import Paste @@ -18,7 +20,25 @@ app = Flask_config.app cfg = Flask_config.cfg max_preview_char = Flask_config.max_preview_char max_preview_modal = Flask_config.max_preview_modal -r_serv_db = Flask_config.r_serv_db + +#init all lvlDB servers +curYear = datetime.now().year +r_serv_db = {} +# port generated automatically depending on available levelDB date +yearList = [] +lvdbdir= os.path.join(os.environ['AIL_HOME'], "LEVEL_DB_DATA/") +for year in os.listdir(lvdbdir): + try: + intYear = int(year) + except: + continue + + yearList.append([year, intYear, int(curYear) == intYear]) + r_serv_db[intYear] = redis.StrictRedis( + host=cfg.get("Redis_Level_DB", "host"), + port=intYear, + db=cfg.getint("Redis_Level_DB", "db")) +yearList.sort(reverse=True) browsepastes = Blueprint('browsepastes', __name__, template_folder='templates') @@ -31,9 +51,9 @@ def getPastebyType(server, module_name): return all_path -def event_stream_getImportantPasteByModule(module_name): +def event_stream_getImportantPasteByModule(module_name, year): index = 0 - all_pastes_list = getPastebyType(r_serv_db, module_name) + all_pastes_list = getPastebyType(r_serv_db[year], module_name) for path in all_pastes_list: index += 1 paste = Paste.Paste(path) @@ -57,18 +77,19 @@ def event_stream_getImportantPasteByModule(module_name): @browsepastes.route("/browseImportantPaste/", methods=['GET']) def browseImportantPaste(): module_name = request.args.get('moduleName') - return render_template("browse_important_paste.html") + return render_template("browse_important_paste.html", year_list=yearList, selected_year=curYear) @browsepastes.route("/importantPasteByModule/", methods=['GET']) def importantPasteByModule(): module_name = request.args.get('moduleName') + currentSelectYear = int(request.args.get('year')) all_content = [] paste_date = [] paste_linenum = [] all_path = [] - allPastes = getPastebyType(r_serv_db, module_name) + allPastes = getPastebyType(r_serv_db[currentSelectYear], module_name) for path in allPastes[0:10]: all_path.append(path) @@ -88,6 +109,7 @@ def importantPasteByModule(): return render_template("important_paste_by_module.html", moduleName=module_name, + year=currentSelectYear, all_path=all_path, content=all_content, paste_date=paste_date, @@ -95,10 +117,11 @@ def importantPasteByModule(): char_to_display=max_preview_modal, finished=finished) -@browsepastes.route("/_getImportantPasteByModule") +@browsepastes.route("/_getImportantPasteByModule", methods=['GET']) def getImportantPasteByModule(): module_name = request.args.get('moduleName') - return flask.Response(event_stream_getImportantPasteByModule(module_name), mimetype="text/event-stream") + currentSelectYear = int(request.args.get('year')) + return flask.Response(event_stream_getImportantPasteByModule(module_name, currentSelectYear), mimetype="text/event-stream") # ========= REGISTRATION ========= diff --git a/var/www/modules/browsepastes/templates/browse_important_paste.html b/var/www/modules/browsepastes/templates/browse_important_paste.html index 5d7d84e3..89fc8aaa 100644 --- a/var/www/modules/browsepastes/templates/browse_important_paste.html +++ b/var/www/modules/browsepastes/templates/browse_important_paste.html @@ -63,6 +63,20 @@ + +
+
+ Year: + +
+
+
+ +
@@ -117,9 +131,10 @@ $("#"+activePage).addClass("active"); var dataPath = 'credential'; - $.get("{{ url_for('browsepastes.importantPasteByModule') }}"+"?moduleName="+dataPath, function(data, status){ + $.get("{{ url_for('browsepastes.importantPasteByModule') }}"+"?moduleName="+dataPath+"&year="+currentSelectYear, function(data, status){ $('#'+dataPath+'-tab').html(data); }); + }); @@ -127,12 +142,23 @@