Merge pull request #136 from mokaddem/bugfix

LevelDB auto-directory
pull/142/merge
Alexandre Dulaunoy 2017-08-24 09:39:58 +02:00 committed by GitHub
commit baab19d663
14 changed files with 180 additions and 101 deletions

View File

@ -28,9 +28,11 @@ if __name__ == "__main__":
p = Process(config_section)
# port generated automatically depending on the date
curYear = datetime.now().year
server = redis.StrictRedis(
host=p.config.get("Redis_Level_DB", "host"),
port=p.config.get("Redis_Level_DB", "port"),
port=curYear,
db=p.config.get("Redis_Level_DB", "db"))
# FUNCTIONS #

View File

@ -65,33 +65,26 @@ function launching_redis {
}
function launching_lvldb {
#Want to launch more level_db?
lvdbhost='127.0.0.1'
lvdbdir="${AIL_HOME}/LEVEL_DB_DATA/"
db1_y='2013'
db2_y='2014'
db3_y='2016'
db4_y='2017'
dbC_y='3016'
nb_db=13
db_y=`date +%Y`
#Verify that a dir with the correct year exists, create it otherwise
if [ ! -d "$lvdbdir$db_y" ]; then
mkdir -p "$db_y"
fi
screen -dmS "LevelDB"
sleep 0.1
echo -e $GREEN"\t* Launching Levels DB servers"$DEFAULT
#Add lines here with appropriates options.
screen -S "LevelDB" -X screen -t "2013" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2013/ -P '$db1_y' -M '$nb_db'; read x'
sleep 0.1
screen -S "LevelDB" -X screen -t "2014" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2014/ -P '$db2_y' -M '$nb_db'; read x'
sleep 0.1
screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db3_y' -M '$nb_db'; read x'
sleep 0.1
screen -S "LevelDB" -X screen -t "2017" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2017/ -P '$db4_y' -M '$nb_db'; read x'
# For Curve
sleep 0.1
screen -S "LevelDB" -X screen -t "3016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'3016/ -P '$dbC_y' -M '$nb_db'; read x'
#Launch a DB for each dir
for pathDir in $lvdbdir*/ ; do
yDir=$(basename "$pathDir")
sleep 0.1
screen -S "LevelDB" -X screen -t "$yDir" bash -c 'redis-leveldb -H '$lvdbhost' -D '$pathDir'/ -P '$yDir' -M '$nb_db'; read x'
done
}
function launching_logs {
@ -255,6 +248,7 @@ for i in ${!options[@]}; do
Killall)
if [[ $isredis || $islvldb || $islogged || $isqueued || $isscripted ]]; then
kill $isredis $islvldb $islogged $isqueued $isscripted
sleep 0.2
echo -e $ROSE`screen -ls`$DEFAULT
echo -e $GREEN"\t* $isredis $islvldb $islogged $isqueued $isscripted killed."$DEFAULT
else

View File

@ -4,6 +4,7 @@
import redis
import argparse
import ConfigParser
from datetime import datetime
from pubsublogger import publisher
import matplotlib.pyplot as plt
@ -24,13 +25,16 @@ def main():
parser.add_argument('-f', type=str, metavar="filename", default="figure",
help='The absolute path name of the "figure.png"',
action='store')
parser.add_argument('-y', '--year', type=int, required=False, default=None, help='The date related to the DB')
args = parser.parse_args()
# REDIS #
# port generated automatically depending on the date
curYear = datetime.now().year if args.year is None else args.year
r_serv = redis.StrictRedis(
host=cfg.get("Redis_Level_DB_Hashs", "host"),
port=cfg.getint("Redis_Level_DB_Hashs", "port"),
port=curYear,
db=cfg.getint("Redis_Level_DB_Hashs", "db"))
# LOGGING #

View File

@ -3,6 +3,10 @@
set -e
set -x
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_LEVELDB" ] && echo "Needs the env var AIL_LEVELDB. Run the script from the virtual environment." && exit 1;
screen -dmS "Logging"
sleep 0.1
echo -e $GREEN"\t* Launching logging process"$DEFAULT

View File

@ -3,21 +3,27 @@
set -e
set -x
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_LEVELDB" ] && echo "Needs the env var AIL_LEVELDB. Run the script from the virtual environment." && exit 1;
lvdbhost='127.0.0.1'
lvdbdir="${AIL_HOME}/LEVEL_DB_DATA/"
db1_y='2013'
db2_y='2014'
db3_y='2016'
db4_y='3016'
nb_db=13
db_y=`date +%Y`
#Verify that a dir with the correct year exists, create it otherwise
if [ ! -d "$lvdbdir$db_y" ]; then
mkdir -p "$db_y"
fi
screen -dmS "LevelDB"
sleep 0.1
echo -e $GREEN"\t* Launching Levels DB servers"$DEFAULT
sleep 0.1
screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db3_y' -M '$nb_db'; read x'
# For Curve
sleep 0.1
screen -S "LevelDB" -X screen -t "3016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'3016/ -P '$db4_y' -M '$nb_db'; read x'
#Launch a DB for each dir
for pathDir in $lvdbdir*/ ; do
yDir=$(basename "$pathDir")
sleep 0.1
screen -S "LevelDB" -X screen -t "$yDir" bash -c 'redis-leveldb -H '$lvdbhost' -D '$pathDir'/ -P '$yDir' -M '$nb_db'; read x'
done

View File

@ -3,6 +3,10 @@
set -e
set -x
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_LEVELDB" ] && echo "Needs the env var AIL_LEVELDB. Run the script from the virtual environment." && exit 1;
screen -dmS "Queue"
sleep 0.1

View File

@ -3,6 +3,10 @@
set -e
set -x
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_LEVELDB" ] && echo "Needs the env var AIL_LEVELDB. Run the script from the virtual environment." && exit 1;
conf_dir="${AIL_HOME}/configs/"
screen -dmS "Redis"

View File

@ -3,57 +3,75 @@
set -e
set -x
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_LEVELDB" ] && echo "Needs the env var AIL_LEVELDB. Run the script from the virtual environment." && exit 1;
echo -e "\t* Checking configuration"
bash -c "./Update-conf.py"
exitStatus=$?
if [ $exitStatus -ge 1 ]; then
echo -e $RED"\t* Configuration not up-to-date"$DEFAULT
exit
fi
echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT
screen -dmS "Script"
sleep 0.1
echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModuleInformation.py -k 0 -c 1; read x'
sleep 0.1
screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Duplicates" bash -c './Duplicates.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Attributes" bash -c './Attributes.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Lines" bash -c './Lines.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "DomClassifier" bash -c './DomClassifier.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Categ" bash -c './Categ.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Tokenize" bash -c './Tokenize.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "CreditCards" bash -c './CreditCards.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Onion" bash -c './Onion.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Mail" bash -c './Mail.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Web" bash -c './Web.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Credential" bash -c './Credential.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "CurveManageTopSets" bash -c './CurveManageTopSets.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Keys" bash -c './Keys.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Phone" bash -c './Phone.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Release" bash -c './Release.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Cve" bash -c './Cve.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "ModuleStats" bash -c './ModuleStats.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Browse_warning_paste" bash -c './Browse_warning_paste.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x'
screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModulesInformationV2.py -k 0 -c 1; read x'
sleep 0.1
screen -S "Script" -X screen -t "Mixer" bash -c './Mixer.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Duplicates" bash -c './Duplicates.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Attributes" bash -c './Attributes.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Lines" bash -c './Lines.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "DomClassifier" bash -c './DomClassifier.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Categ" bash -c './Categ.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Tokenize" bash -c './Tokenize.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "CreditCards" bash -c './CreditCards.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Onion" bash -c './Onion.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Mail" bash -c './Mail.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Web" bash -c './Web.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Credential" bash -c './Credential.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "CurveManageTopSets" bash -c './CurveManageTopSets.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "RegexForTermsFrequency" bash -c './RegexForTermsFrequency.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "SetForTermsFrequency" bash -c './SetForTermsFrequency.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Keys" bash -c './Keys.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Phone" bash -c './Phone.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Release" bash -c './Release.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Cve" bash -c './Cve.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "ModuleStats" bash -c './ModuleStats.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "BrowseWarningPaste" bash -c './BrowseWarningPaste.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x'

View File

@ -104,7 +104,6 @@ db = 5
[Redis_Level_DB]
host = localhost
port = 2016
db = 0
[Redis_Level_DB_Trending]
@ -114,7 +113,6 @@ db = 3
[Redis_Level_DB_Hashs]
host = localhost
port = 2016
db = 1
[Url]

View File

@ -84,9 +84,9 @@ if [ -z "$VIRTUAL_ENV" ]; then
fi
year1=20`date +%y`
mkdir -p $AIL_HOME/{PASTES,Blooms,dumps}
mkdir -p $AIL_HOME/LEVEL_DB_DATA/2016
mkdir -p $AIL_HOME/LEVEL_DB_DATA/3016
mkdir -p $AIL_HOME/LEVEL_DB_DATA/$year1
pip install -U pip
pip install -U -r pip_packages_requirement.txt

View File

@ -38,11 +38,6 @@ r_serv_charts = redis.StrictRedis(
port=cfg.getint("Redis_Level_DB_Trending", "port"),
db=cfg.getint("Redis_Level_DB_Trending", "db"))
r_serv_db = redis.StrictRedis(
host=cfg.get("Redis_Level_DB", "host"),
port=cfg.getint("Redis_Level_DB", "port"),
db=cfg.getint("Redis_Level_DB", "db"))
r_serv_sentiment = redis.StrictRedis(
host=cfg.get("Redis_Level_DB_Sentiment", "host"),
port=cfg.getint("Redis_Level_DB_Sentiment", "port"),

View File

@ -7,6 +7,8 @@
import redis
import json
import flask
import os
from datetime import datetime
from flask import Flask, render_template, jsonify, request, Blueprint
import Paste
@ -18,7 +20,25 @@ app = Flask_config.app
cfg = Flask_config.cfg
max_preview_char = Flask_config.max_preview_char
max_preview_modal = Flask_config.max_preview_modal
r_serv_db = Flask_config.r_serv_db
#init all lvlDB servers
curYear = datetime.now().year
r_serv_db = {}
# port generated automatically depending on available levelDB date
yearList = []
lvdbdir= os.path.join(os.environ['AIL_HOME'], "LEVEL_DB_DATA/")
for year in os.listdir(lvdbdir):
try:
intYear = int(year)
except:
continue
yearList.append([year, intYear, int(curYear) == intYear])
r_serv_db[intYear] = redis.StrictRedis(
host=cfg.get("Redis_Level_DB", "host"),
port=intYear,
db=cfg.getint("Redis_Level_DB", "db"))
yearList.sort(reverse=True)
browsepastes = Blueprint('browsepastes', __name__, template_folder='templates')
@ -31,9 +51,9 @@ def getPastebyType(server, module_name):
return all_path
def event_stream_getImportantPasteByModule(module_name):
def event_stream_getImportantPasteByModule(module_name, year):
index = 0
all_pastes_list = getPastebyType(r_serv_db, module_name)
all_pastes_list = getPastebyType(r_serv_db[year], module_name)
for path in all_pastes_list:
index += 1
paste = Paste.Paste(path)
@ -57,18 +77,19 @@ def event_stream_getImportantPasteByModule(module_name):
@browsepastes.route("/browseImportantPaste/", methods=['GET'])
def browseImportantPaste():
module_name = request.args.get('moduleName')
return render_template("browse_important_paste.html")
return render_template("browse_important_paste.html", year_list=yearList, selected_year=curYear)
@browsepastes.route("/importantPasteByModule/", methods=['GET'])
def importantPasteByModule():
module_name = request.args.get('moduleName')
currentSelectYear = int(request.args.get('year'))
all_content = []
paste_date = []
paste_linenum = []
all_path = []
allPastes = getPastebyType(r_serv_db, module_name)
allPastes = getPastebyType(r_serv_db[currentSelectYear], module_name)
for path in allPastes[0:10]:
all_path.append(path)
@ -88,6 +109,7 @@ def importantPasteByModule():
return render_template("important_paste_by_module.html",
moduleName=module_name,
year=currentSelectYear,
all_path=all_path,
content=all_content,
paste_date=paste_date,
@ -95,10 +117,11 @@ def importantPasteByModule():
char_to_display=max_preview_modal,
finished=finished)
@browsepastes.route("/_getImportantPasteByModule")
@browsepastes.route("/_getImportantPasteByModule", methods=['GET'])
def getImportantPasteByModule():
module_name = request.args.get('moduleName')
return flask.Response(event_stream_getImportantPasteByModule(module_name), mimetype="text/event-stream")
currentSelectYear = int(request.args.get('year'))
return flask.Response(event_stream_getImportantPasteByModule(module_name, currentSelectYear), mimetype="text/event-stream")
# ========= REGISTRATION =========

View File

@ -63,6 +63,20 @@
</div>
<!-- /.col-lg-12 -->
</div>
<div class="row">
<div class="col-md-12" style="margin-bottom: 0.2cm;">
<strong style="">Year: </strong>
<select class="form-control" id="index_year" style="display: inline-block; margin-bottom: 5px; width: 5%">
{% for yearElem in year_list %}
<option {% if yearElem[2] %} selected="selected" {% endif %} value="{{ yearElem[0] }}" >{{ yearElem[1] }}</option>
{% endfor %}
</select>
</div>
</br>
</div>
<!-- /.row -->
<div class="row">
@ -117,9 +131,10 @@
$("#"+activePage).addClass("active");
var dataPath = 'credential';
$.get("{{ url_for('browsepastes.importantPasteByModule') }}"+"?moduleName="+dataPath, function(data, status){
$.get("{{ url_for('browsepastes.importantPasteByModule') }}"+"?moduleName="+dataPath+"&year="+currentSelectYear, function(data, status){
$('#'+dataPath+'-tab').html(data);
});
});
</script>
@ -127,12 +142,23 @@
<script>
// When a pannel is shown, create the data-table.
var previous_tab = $('[data-attribute-name="credential');
var currentTabName = previous_tab.attr('data-attribute-name');
var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_for('static', filename='image/loading.gif') }}\" height='26' width='26' style='margin: 4px;'>";
var currentSelectYear = {{ selected_year }};
$('#index_year').on('change', function() {
currentSelectYear = this.value;
$.get("{{ url_for('browsepastes.importantPasteByModule') }}"+"?moduleName="+currentTabName+"&year="+currentSelectYear, function(data, status){
$('#'+currentTabName+'-tab').html(data);
});
})
$('.nav-tabs a').on('shown.bs.tab', function(event){
var dataPath = $(event.target).attr('data-attribute-name');
$.get("{{ url_for('browsepastes.importantPasteByModule') }}"+"?moduleName="+dataPath, function(data, status){
var currentTab = $('[name].active').children();
currentTabName = dataPath;
$.get("{{ url_for('browsepastes.importantPasteByModule') }}"+"?moduleName="+currentTabName+"&year="+currentSelectYear, function(data, status){
currentTab = $('[name].active').children();
$('#'+previous_tab.attr('data-attribute-name')+'-tab').html(loading_gif);
currentTab.removeClass( "active" );

View File

@ -45,7 +45,7 @@ function deploy_source() {
if(typeof(EventSource) !== "undefined" && typeof(source) !== "") {
$("#load_more_json_button1").show();
$("#load_more_json_button2").show();
var source = new EventSource("{{ url_for('browsepastes.getImportantPasteByModule') }}"+"?moduleName="+moduleName);
var source = new EventSource("{{ url_for('browsepastes.getImportantPasteByModule') }}"+"?moduleName="+moduleName+"&year="+currentSelectYear);
source.onmessage = function(event) {
var feed = jQuery.parseJSON( event.data );
curr_numElem = parseInt($("#myTable_"+moduleName).attr('data-numElem'));
@ -113,6 +113,7 @@ function add_entries_X(to_add) {
<script>
var moduleName = "{{ moduleName }}";
var currentSelectYear = "{{ year }}";
var search_table;
var last_clicked_paste;
var can_change_modal_content = true;