mirror of https://github.com/CIRCL/AIL-framework
Finished basic link web interface + solved some bugs
Mokaddem
parent
ef741f85c0
commit
05417f405c
|
|
@ -18,7 +18,7 @@ Redis organization:
|
|||
AllCredentialsRev: the opposite of AllCredentials, uniqNum -> username
|
||||
AllPath: hashed set where keys are path and value are their uniq number
|
||||
AllPathRev: the opposite of AllPath, uniqNum -> path
|
||||
splitedCred -> uniq_num (set)
|
||||
CredToPathMapping_uniqNumForUsername -> (set) -> uniqNumForPath
|
||||
|
||||
"""
|
||||
|
||||
|
|
@ -54,7 +54,7 @@ if __name__ == "__main__":
|
|||
host=p.config.get("Redis_Level_DB_TermCred", "host"),
|
||||
port=p.config.get("Redis_Level_DB_TermCred", "port"),
|
||||
db=p.config.get("Redis_Level_DB_TermCred", "db"))
|
||||
Credential
|
||||
|
||||
critical = 8
|
||||
|
||||
regex_web = "((?:https?:\/\/)[-_0-9a-zA-Z]+\.[0-9a-zA-Z]+)"
|
||||
|
|
@ -136,7 +136,6 @@ Credential
|
|||
|
||||
#unique number attached to unique path
|
||||
uniq_num_path = server_cred.incr(REDIS_KEY_NUM_PATH)
|
||||
print(REDIS_KEY_ALL_PATH_SET, {filepath: uniq_num_path})
|
||||
server_cred.hmset(REDIS_KEY_ALL_PATH_SET, {filepath: uniq_num_path})
|
||||
server_cred.hmset(REDIS_KEY_ALL_PATH_SET_REV, {uniq_num_path: filepath})
|
||||
|
||||
|
|
@ -147,10 +146,10 @@ Credential
|
|||
server_cred.hmset(REDIS_KEY_ALL_CRED_SET, {cred: uniq_num_cred})
|
||||
server_cred.hmset(REDIS_KEY_ALL_CRED_SET_REV, {uniq_num_cred: cred})
|
||||
|
||||
server_cred.hmset(REDIS_KEY_MAP_CRED_TO_PATH, {uniq_num_cred: uniq_num_path})
|
||||
#server_cred.hmset(REDIS_KEY_MAP_CRED_TO_PATH, {uniq_num_cred: uniq_num_path})
|
||||
server_cred.sadd(REDIS_KEY_MAP_CRED_TO_PATH+'_'+str(uniq_num_cred), uniq_num_path)
|
||||
|
||||
splitedCred = re.findall(REGEX_CRED, cred)
|
||||
print(splitedCred)
|
||||
for partCred in splitedCred:
|
||||
if len(partCred) > MINIMUMSIZETHRESHOLD:
|
||||
server_cred.sadd(partCred, uniq_num_cred)
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ r_serv_cred = Flask_config.r_serv_cred
|
|||
|
||||
terms = Blueprint('terms', __name__, template_folder='templates')
|
||||
|
||||
'''TERM'''
|
||||
DEFAULT_MATCH_PERCENT = 50
|
||||
|
||||
#tracked
|
||||
|
|
@ -39,6 +40,19 @@ TrackedRegexDate_Name = "TrackedRegexDate"
|
|||
TrackedSetSet_Name = "TrackedSetSet"
|
||||
TrackedSetDate_Name = "TrackedSetDate"
|
||||
|
||||
|
||||
'''CRED'''
|
||||
REGEX_CRED = '[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+'
|
||||
REDIS_KEY_NUM_USERNAME = 'uniqNumForUsername'
|
||||
REDIS_KEY_NUM_PATH = 'uniqNumForUsername'
|
||||
REDIS_KEY_ALL_CRED_SET = 'AllCredentials'
|
||||
REDIS_KEY_ALL_CRED_SET_REV = 'AllCredentialsRev'
|
||||
REDIS_KEY_ALL_PATH_SET = 'AllPath'
|
||||
REDIS_KEY_ALL_PATH_SET_REV = 'AllPathRev'
|
||||
REDIS_KEY_MAP_CRED_TO_PATH = 'CredToPathMapping'
|
||||
|
||||
|
||||
|
||||
# ============ FUNCTIONS ============
|
||||
|
||||
def Term_getValueOverRange(word, startDate, num_day, per_paste=""):
|
||||
|
|
@ -55,7 +69,7 @@ def Term_getValueOverRange(word, startDate, num_day, per_paste=""):
|
|||
passed_days += 1
|
||||
return to_return
|
||||
|
||||
def mixUserName(supplied):
|
||||
def mixUserName(supplied, extensive=False):
|
||||
#e.g.: John Smith
|
||||
terms = supplied.split()[:2]
|
||||
usernames = []
|
||||
|
|
@ -88,7 +102,20 @@ def mixUserName(supplied):
|
|||
usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
|
||||
usernames += [(terms[0][0].lower() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
|
||||
|
||||
return usernames
|
||||
if not extensive:
|
||||
return usernames
|
||||
|
||||
mixedSupplied = supplied.replace(' ','')
|
||||
minWindow = 3 if len(mixedSupplied)/2 < 4 else len(mixedSupplied)/2
|
||||
for winSize in range(3,len(mixedSupplied)):
|
||||
for startIndex in range(0, len(mixedSupplied)-winSize):
|
||||
usernames += [mixedSupplied[startIndex:startIndex+winSize]]
|
||||
|
||||
filtered_usernames = []
|
||||
for usr in usernames:
|
||||
if len(usr) > 2:
|
||||
filtered_usernames.append(usr)
|
||||
return filtered_usernames
|
||||
|
||||
|
||||
# ============ ROUTES ============
|
||||
|
|
@ -305,6 +332,7 @@ def terms_plot_tool():
|
|||
|
||||
@terms.route("/terms_plot_tool_data/")
|
||||
def terms_plot_tool_data():
|
||||
|
||||
oneDay = 60*60*24
|
||||
range_start = datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_start')))) if request.args.get('range_start') is not None else 0;
|
||||
range_start = range_start.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
|
|
@ -323,6 +351,7 @@ def terms_plot_tool_data():
|
|||
|
||||
if term is None:
|
||||
return "None"
|
||||
|
||||
else:
|
||||
value_range = []
|
||||
for timestamp in range(range_start, range_end+oneDay, oneDay):
|
||||
|
|
@ -335,6 +364,7 @@ def terms_plot_tool_data():
|
|||
|
||||
@terms.route("/terms_plot_top/")
|
||||
def terms_plot_top():
|
||||
|
||||
per_paste = request.args.get('per_paste')
|
||||
per_paste = per_paste if per_paste is not None else 1
|
||||
return render_template("terms_plot_top.html", per_paste=per_paste)
|
||||
|
|
@ -347,6 +377,7 @@ def terms_plot_top_data():
|
|||
today = today.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
today_timestamp = calendar.timegm(today.timetuple())
|
||||
|
||||
|
||||
per_paste = request.args.get('per_paste')
|
||||
if per_paste == "1" or per_paste is None:
|
||||
per_paste = "per_paste_"
|
||||
|
|
@ -390,51 +421,84 @@ def terms_plot_top_data():
|
|||
def credentials_tracker():
|
||||
return render_template("credentials_tracker.html")
|
||||
|
||||
@terms.route("/credentials_management_query_paste/")
|
||||
@terms.route("/credentials_management_query_paste/", methods=['GET', 'POST'])
|
||||
def credentials_management_query_paste():
|
||||
cred = request.args.get('cred')
|
||||
return 1
|
||||
allPath = request.json['allPath']
|
||||
|
||||
|
||||
paste_info = []
|
||||
for pathNum in allPath:
|
||||
path = r_serv_cred.hget(REDIS_KEY_ALL_PATH_SET_REV, pathNum)
|
||||
paste = Paste.Paste(path)
|
||||
p_date = str(paste._get_p_date())
|
||||
p_date = p_date[6:]+'/'+p_date[4:6]+'/'+p_date[0:4]
|
||||
p_source = paste.p_source
|
||||
p_encoding = paste._get_p_encoding()
|
||||
p_size = paste.p_size
|
||||
p_mime = paste.p_mime
|
||||
p_lineinfo = paste.get_lines_info()
|
||||
p_content = paste.get_p_content().decode('utf-8', 'ignore')
|
||||
if p_content != 0:
|
||||
p_content = p_content[0:400]
|
||||
paste_info.append({"path": path, "date": p_date, "source": p_source, "encoding": p_encoding, "size": p_size, "mime": p_mime, "lineinfo": p_lineinfo, "content": p_content})
|
||||
|
||||
return jsonify(paste_info)
|
||||
|
||||
@terms.route("/credentials_management_action/", methods=['GET'])
|
||||
def cred_management_action():
|
||||
REGEX_CRED = '[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+'
|
||||
REDIS_KEY_NUM_USERNAME = 'uniqNumForUsername'
|
||||
REDIS_KEY_NUM_PATH = 'uniqNumForUsername'
|
||||
REDIS_KEY_ALL_CRED_SET = 'AllCredentials'
|
||||
REDIS_KEY_ALL_CRED_SET_REV = 'AllCredentialsRev'
|
||||
REDIS_KEY_ALL_PATH_SET = 'AllPath'
|
||||
REDIS_KEY_ALL_PATH_SET_REV = 'AllPath'
|
||||
REDIS_KEY_MAP_CRED_TO_PATH = 'CredToPathMapping'
|
||||
|
||||
supplied = request.args.get('term').encode('utf-8')
|
||||
action = request.args.get('action')
|
||||
section = request.args.get('section')
|
||||
extensive = request.args.get('extensive')
|
||||
extensive = True if extensive == "true" else False
|
||||
|
||||
#splitedCred = re.findall(REGEX_CRED, cred)
|
||||
if extensive:
|
||||
#collectDico
|
||||
AllUsernameInRedis = r_serv_cred.hgetall(REDIS_KEY_ALL_CRED_SET).keys()
|
||||
uniq_num_set = set()
|
||||
if action == "seek":
|
||||
possibilities = mixUserName(supplied)
|
||||
possibilities = mixUserName(supplied, extensive)
|
||||
for poss in possibilities:
|
||||
num = r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET, poss)
|
||||
if num is not None:
|
||||
uniq_num_set.add(num)
|
||||
for num in r_serv_cred.smembers(poss):
|
||||
uniq_num_set.add(num)
|
||||
#Extensive /!\
|
||||
if extensive:
|
||||
for tempUsername in AllUsernameInRedis:
|
||||
for poss in possibilities:
|
||||
if poss in tempUsername:
|
||||
num = r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET, tempUsername)
|
||||
if num is not None:
|
||||
uniq_num_set.add(num)
|
||||
for num in r_serv_cred.smembers(tempUsername):
|
||||
uniq_num_set.add(num)
|
||||
|
||||
data = {'usr': [], 'path': [], 'numPaste': [], 'simil': []}
|
||||
for Unum in uniq_num_set:
|
||||
levenRatio = 2.0
|
||||
username = r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET_REV, Unum)
|
||||
|
||||
# Calculate Levenshtein distance, ignore negative ratio
|
||||
levenDist = float(Levenshtein.distance(supplied, username))
|
||||
levenRatio = levenDist / float(len(supplied))
|
||||
levenRatioStr = "{:.1%}".format(1.0 - levenRatio)
|
||||
if levenRatio >= 1.0:
|
||||
continue
|
||||
supp_splitted = supplied.split()
|
||||
supp_mixed = supplied.replace(' ','')
|
||||
supp_splitted.append(supp_mixed)
|
||||
for indiv_supplied in supp_splitted:
|
||||
#levenDist = float(Levenshtein.distance(indiv_supplied, username))
|
||||
#levenRatio = levenDist / float(len(indiv_supplied)) if levenRatio > levenDist / float(len(indiv_supplied)) else levenRatio
|
||||
#levenRatio = levenRatio if levenRatio < 1.0 else 1.0
|
||||
levenRatio = float(Levenshtein.ratio(indiv_supplied, username))
|
||||
levenRatioStr = "{:.1%}".format(levenRatio)
|
||||
#levenRatioStr = "{:.1%}".format(1.0 - levenRatio)
|
||||
#if levenRatio >= 1.0:
|
||||
# continue
|
||||
|
||||
data['usr'].append(username)
|
||||
data['path'].append(r_serv_cred.hget(REDIS_KEY_MAP_CRED_TO_PATH, Unum))
|
||||
data['numPaste'].append(len(uniq_num_set))
|
||||
allPathNum = list(r_serv_cred.smembers(REDIS_KEY_MAP_CRED_TO_PATH+'_'+Unum))
|
||||
data['path'].append(allPathNum)
|
||||
data['numPaste'].append(len(allPathNum))
|
||||
data['simil'].append(levenRatioStr)
|
||||
|
||||
to_return = {}
|
||||
|
|
|
|||
|
|
@ -69,6 +69,7 @@
|
|||
</div>
|
||||
<div class="panel-body">
|
||||
|
||||
<label><input type="checkbox" id="extensive" style="margin-bottom: 10px;"><strong>Extensive search (takes time)</strong></input></label>
|
||||
<div class="form-group input-group" style="margin-bottom: 30px;">
|
||||
<span class="input-group-addon"><span class="glyphicon glyphicon-screenshot"></span></span>
|
||||
<input id="seekInput" class="form-control" placeholder="Credential to seek." type="text" style="max-width: 400px;">
|
||||
|
|
@ -123,13 +124,33 @@
|
|||
|
||||
perform_binding();
|
||||
|
||||
$("#mymodal").on('hidden.bs.modal', function () {
|
||||
$("#mymodalbody").html("<p>Loading paste information...</p>");
|
||||
var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_for('static', filename='image/loading.gif') }}\" height='26' width='26' style='margin: 4px;'>";
|
||||
$("#mymodalbody").append(loading_gif); // Show the loading GIF
|
||||
});
|
||||
});
|
||||
</script>
|
||||
|
||||
<script>
|
||||
function perform_binding() {
|
||||
$(".btn-interaction").unbind("click.interaction");
|
||||
$(".btn-interaction").bind("click.interaction", perform_operation);
|
||||
}
|
||||
|
||||
function perform_modal_binding() {
|
||||
// On click, get html content from url and update the corresponding modal
|
||||
$("[data-toggle='modal']").on("click.openmodal", function (event) {
|
||||
//console.log(data);
|
||||
event.preventDefault();
|
||||
var the_modal=$(this);
|
||||
var url = "{{ url_for('terms.credentials_management_query_paste') }}?cred=" + encodeURIComponent($(this).attr('data-term'));
|
||||
$.getJSON(url, function (data) {
|
||||
$.ajax({
|
||||
type: 'POST',
|
||||
url: url,
|
||||
dataType: "json",
|
||||
data: JSON.stringify({ 'allPath': JSON.parse($(this).attr('data-path')) }),
|
||||
contentType : "application/json"
|
||||
}).done(function (data) {
|
||||
if (data.length != 0) {
|
||||
var html_to_add = "";
|
||||
html_to_add += "<table id=\"modal-table\" class=\"table table-striped\">";
|
||||
|
|
@ -166,22 +187,8 @@
|
|||
} else {
|
||||
$("#mymodalbody").html("No paste containing this term has been received yet.");
|
||||
}
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
$("#mymodal").on('hidden.bs.modal', function () {
|
||||
$("#mymodalbody").html("<p>Loading paste information...</p>");
|
||||
var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_for('static', filename='image/loading.gif') }}\" height='26' width='26' style='margin: 4px;'>";
|
||||
$("#mymodalbody").append(loading_gif); // Show the loading GIF
|
||||
});
|
||||
});
|
||||
</script>
|
||||
|
||||
<script>
|
||||
function perform_binding() {
|
||||
$(".btn-interaction").unbind("click.interaction");
|
||||
$(".btn-interaction").bind("click.interaction", perform_operation);
|
||||
}
|
||||
)});
|
||||
}
|
||||
|
||||
function perform_operation(){
|
||||
|
|
@ -194,7 +201,7 @@
|
|||
} else {
|
||||
var curr_term = $(this).attr('data-content');
|
||||
}
|
||||
var data_to_send = { section: curr_section, action:curr_action, term: curr_term};
|
||||
var data_to_send = { section: curr_section, action:curr_action, term: curr_term, extensive: $("#extensive").is(":checked")};
|
||||
|
||||
if (curr_term != "") {
|
||||
//console.log(data_to_send);
|
||||
|
|
@ -203,13 +210,14 @@
|
|||
var json = data;
|
||||
|
||||
if(json.action == "add") {
|
||||
//not used for the moment
|
||||
|
||||
} else if (json.action == "seek") {
|
||||
table_track.clear().draw();
|
||||
var rep = json.data;
|
||||
var action_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"left\" title=\"Track this term\" data-content=\"" + json.term + "\" data-section=\"followTerm\" data-action=\"add\">";
|
||||
//var action_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"left\" title=\"Track this term\" data-content=\"" + json.term + "\" data-section=\"followTerm\" data-action=\"add\">";
|
||||
var action_button = "";
|
||||
if (rep.usr.length == 0) {
|
||||
console.log('noData');
|
||||
$( "#nodata" ).removeClass('alert-info').addClass('alert-danger');
|
||||
$( "#nodata" ).text("No data");
|
||||
$( "#nodata" ).fadeIn( "fast", function() {
|
||||
|
|
@ -218,13 +226,11 @@
|
|||
}, 2000);
|
||||
});
|
||||
} else {
|
||||
console.log(json);
|
||||
console.log(table_track);
|
||||
for(i=0; i < rep.usr.length; i++) {
|
||||
$( "#nodata" ).removeClass('alert-danger').addClass('alert-info');
|
||||
$( "#nodata" ).text(curr_term);
|
||||
$( "#nodata" ).fadeIn( "fast");
|
||||
toAdd = "<span class=\"fa fa-eye\"></span></button><span data-toggle=\"modal\" data-target=\"#mymodal\" data-term=\""+rep.usr[i]+"\" ><button class=\"btn-link\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Show concerned paste(s)\"><span class=\"glyphicon glyphicon-info-sign\"></span></button></span>";
|
||||
toAdd = "</button><span data-toggle=\"modal\" data-target=\"#mymodal\" data-term=\""+rep.usr[i]+"\" data-path=\"["+rep.path[i]+"]\" ><button class=\"btn-link\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Show concerned paste(s)\"><span class=\"glyphicon glyphicon-info-sign\"></span></button></span>";
|
||||
table_track.row.add( [
|
||||
rep.usr[i],
|
||||
rep.simil[i],
|
||||
|
|
@ -232,6 +238,7 @@
|
|||
toAdd+action_button ] ).draw( false );
|
||||
}
|
||||
perform_binding();
|
||||
perform_modal_binding();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue