Added similarity checks + started link webInterface

pull/131/head
Mokaddem 2017-07-19 11:52:06 +02:00
parent 882ab88b00
commit ef741f85c0
6 changed files with 146 additions and 112 deletions

View File

@ -40,6 +40,7 @@ REDIS_KEY_ALL_CRED_SET_REV = 'AllCredentialsRev'
REDIS_KEY_ALL_PATH_SET = 'AllPath'
REDIS_KEY_ALL_PATH_SET_REV = 'AllPathRev'
REDIS_KEY_MAP_CRED_TO_PATH = 'CredToPathMapping'
MINIMUMSIZETHRESHOLD = 3
if __name__ == "__main__":
publisher.port = 6380
@ -53,7 +54,7 @@ if __name__ == "__main__":
host=p.config.get("Redis_Level_DB_TermCred", "host"),
port=p.config.get("Redis_Level_DB_TermCred", "port"),
db=p.config.get("Redis_Level_DB_TermCred", "db"))
Credential
critical = 8
regex_web = "((?:https?:\/\/)[-_0-9a-zA-Z]+\.[0-9a-zA-Z]+)"
@ -134,7 +135,7 @@ if __name__ == "__main__":
cred = cred.split('@')[0]
#unique number attached to unique path
uniq_num_path = server_cred.incr(REDIS_KEY_ALL_PATH_SET)
uniq_num_path = server_cred.incr(REDIS_KEY_NUM_PATH)
print(REDIS_KEY_ALL_PATH_SET, {filepath: uniq_num_path})
server_cred.hmset(REDIS_KEY_ALL_PATH_SET, {filepath: uniq_num_path})
server_cred.hmset(REDIS_KEY_ALL_PATH_SET_REV, {uniq_num_path: filepath})
@ -151,6 +152,7 @@ if __name__ == "__main__":
splitedCred = re.findall(REGEX_CRED, cred)
print(splitedCred)
for partCred in splitedCred:
server_cred.sadd(partCred, uniq_num_cred)
if len(partCred) > MINIMUMSIZETHRESHOLD:
server_cred.sadd(partCred, uniq_num_cred)

View File

@ -21,6 +21,7 @@ nltk
crcmod
mmh3
ssdeep
python-Levenshtein
#Others
python-magic

View File

@ -12,6 +12,7 @@ from flask import Flask, render_template, jsonify, request, Blueprint
import re
import Paste
from pprint import pprint
import Levenshtein
# ============ VARIABLES ============
import Flask_config
@ -54,6 +55,41 @@ def Term_getValueOverRange(word, startDate, num_day, per_paste=""):
passed_days += 1
return to_return
def mixUserName(supplied):
#e.g.: John Smith
terms = supplied.split()[:2]
usernames = []
if len(terms) == 1:
terms.append(' ')
#john, smith, John, Smith, JOHN, SMITH
usernames += [terms[0].lower()]
usernames += [terms[1].lower()]
usernames += [terms[0][0].upper() + terms[0][1:].lower()]
usernames += [terms[1][0].upper() + terms[1][1:].lower()]
usernames += [terms[0].upper()]
usernames += [terms[1].upper()]
#johnsmith, smithjohn, JOHNsmith, johnSMITH, SMITHjohn, smithJOHN
usernames += [(terms[0].lower() + terms[1].lower()).strip()]
usernames += [(terms[1].lower() + terms[0].lower()).strip()]
usernames += [(terms[0].upper() + terms[1].lower()).strip()]
usernames += [(terms[0].lower() + terms[1].upper()).strip()]
usernames += [(terms[1].upper() + terms[0].lower()).strip()]
usernames += [(terms[1].lower() + terms[0].upper()).strip()]
#Jsmith, JSmith, jsmith, jSmith, johnS, Js, JohnSmith, Johnsmith, johnSmith
usernames += [(terms[0][0].upper() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
usernames += [(terms[0][0].upper() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
usernames += [(terms[0][0].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
usernames += [(terms[0][0].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
usernames += [(terms[0].lower() + terms[1][0].upper()).strip()]
usernames += [(terms[0].upper() + terms[1][0].lower()).strip()]
usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
usernames += [(terms[0][0].lower() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
return usernames
# ============ ROUTES ============
@ -359,41 +395,7 @@ def credentials_management_query_paste():
cred = request.args.get('cred')
return 1
def mixUserName(supplied):
#e.g.: John Smith
terms = supplied.split()[:2]
usernames = []
if len(terms) == 1:
terms.append(' ')
#john, smith, John, Smith, JOHN, SMITH
usernames += [terms[0].lower()]
usernames += [terms[1].lower()]
usernames += [terms[0][0].upper() + terms[0][1:].lower()]
usernames += [terms[1][0].upper() + terms[1][1:].lower()]
usernames += [terms[0].upper()]
usernames += [terms[1].upper()]
#johnsmith, smithjohn, JOHNsmith, johnSMITH, SMITHjohn, smithJOHN
usernames += [(terms[0].lower() + terms[1].lower()).strip()]
usernames += [(terms[1].lower() + terms[0].lower()).strip()]
usernames += [(terms[0].upper() + terms[1].lower()).strip()]
usernames += [(terms[0].lower() + terms[1].upper()).strip()]
usernames += [(terms[1].upper() + terms[0].lower()).strip()]
usernames += [(terms[1].lower() + terms[0].upper()).strip()]
#Jsmith, JSmith, jsmith, jSmith, johnS, Js, JohnSmith, Johnsmith, johnSmith
usernames += [(terms[0][0].upper() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
usernames += [(terms[0][0].upper() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
usernames += [(terms[0][0].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
usernames += [(terms[0][0].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
usernames += [(terms[0].lower() + terms[1][0].upper()).strip()]
usernames += [(terms[0].upper() + terms[1][0].lower()).strip()]
usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
usernames += [(terms[0][0].lower() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
return usernames
@terms.route("/credentials_management_action/", methods=['GET'])
@ -407,7 +409,7 @@ def cred_management_action():
REDIS_KEY_ALL_PATH_SET_REV = 'AllPath'
REDIS_KEY_MAP_CRED_TO_PATH = 'CredToPathMapping'
supplied = request.args.get('term')
supplied = request.args.get('term').encode('utf-8')
action = request.args.get('action')
section = request.args.get('section')
@ -419,12 +421,22 @@ def cred_management_action():
for num in r_serv_cred.smembers(poss):
uniq_num_set.add(num)
data = {'usr': [], 'path': []}
data = {'usr': [], 'path': [], 'numPaste': [], 'simil': []}
for Unum in uniq_num_set:
data['usr'].append(r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET_REV, Unum))
data['path'].append(r_serv_cred.hget(REDIS_KEY_MAP_CRED_TO_PATH, Unum))
username = r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET_REV, Unum)
# Calculate Levenshtein distance, ignore negative ratio
levenDist = float(Levenshtein.distance(supplied, username))
levenRatio = levenDist / float(len(supplied))
levenRatioStr = "{:.1%}".format(1.0 - levenRatio)
if levenRatio >= 1.0:
continue
data['usr'].append(username)
data['path'].append(r_serv_cred.hget(REDIS_KEY_MAP_CRED_TO_PATH, Unum))
data['numPaste'].append(len(uniq_num_set))
data['simil'].append(levenRatioStr)
pprint(data)
to_return = {}
to_return["section"] = section
to_return["action"] = action

View File

@ -1,40 +1,62 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Analysis Information Leak framework Dashboard</title>
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
<!-- JS -->
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.pie.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
</head>
<body>
{% include 'navbar.html' %}
<div id="page-wrapper">
<div class="row">
<div class="col-lg-12">
<h1 class="page-header" data-page="page-termsfrequency" >Credential seeker</h1>
</div>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Analysis Information Leak framework Dashboard</title>
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
<!-- JS -->
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.pie.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
<style>
.btn-link {
color: #000000
}
</style>
</head>
<body>
<!-- Modal -->
<div id="mymodal" class="modal fade" role="dialog">
<div class="modal-dialog modal-lg">
<!-- Modal content-->
<div id="mymodalcontent" class="modal-content">
<div id="mymodalbody" class="modal-body" max-width="8500px">
<p>Loading paste information...</p>
<img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" height="26" width="26" style="margin: 4px;">
</div>
<div class="modal-footer">
<a id="button_show_plot" target="_blank" href=""><button type="button" class="btn btn-info">Plot term</button></a>
<button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
</div>
</div>
</div>
</div>
{% include 'navbar.html' %}
<div id="page-wrapper">
<div class="row">
<div class="col-lg-12">
<h1 class="page-header" data-page="page-termsfrequency" >Credential seeker</h1>
</div>
<!-- /.col-lg-12 -->
<!-- Panel OPTIONS -->
<div class="row">
@ -48,36 +70,22 @@
<div class="panel-body">
<div class="form-group input-group" style="margin-bottom: 30px;">
<span class="input-group-addon"><span class="fa fa-eye"></span></span>
<span class="input-group-addon"><span class="glyphicon glyphicon-screenshot"></span></span>
<input id="seekInput" class="form-control" placeholder="Credential to seek." type="text" style="max-width: 400px;">
<button id="followTermBtn" class="btn btn-success btn-interaction" style="margin-left: 10px;" data-section="seek" data-action="seek"> Seek</button>
<span id="nodata" class="alert alert-danger" style="margin-left: 10px; display: none; padding: 6px;">No data</span>
<span id="nodata" class="alert alert-info" style="margin-left: 10px; display: none; padding: 6px;">No data</span>
</div>
<table class="table table-striped table-bordered table-hover" id="myTable">
<thead>
<tr>
<th style="max-width: 800px;">Credential</th>
<th>Date</th>
<th># line in the paste</th>
<th># tracked paste</th>
<th style="max-width: 800px;">Username</th>
<th>Similarity</th>
<th># concerned paste(s)</th>
<th>Action</th>
</tr>
</thead>
<tbody>
<!-- SET -->
{% for set in trackSet_list %}
<tr style="background-color: #cdffca;">
<td>{{ set }}</td>
<td>{{ trackSet_list_values[loop.index0][0] }}</td>
<td>{{ trackSet_list_values[loop.index0][1] }}</td>
<td>{{ trackSet_list_num_of_paste[loop.index0] }}</td>
<td><p style="margin: 0px;">
<span data-toggle="modal" data-target="#mymodal" data-term="{{ set }}" ><button class="btn-link" data-toggle="tooltip" data-placement="right" title="Show concerned paste(s)"><span class="glyphicon glyphicon-info-sign"></span></button></span>
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Track this credentials" data-content="{{ set }}" data-section="followTerm" data-action="add"><span class="fa fa-eye"></span></button>
</p></td>
</tr>
{% endfor %}
</tbody>
</table>
<!-- /.panel-body -->
@ -88,9 +96,10 @@
<!-- /.panel -->
</div>
</div>
</div>
<!-- /.row -->
</div>
</div>
<!-- /.row -->
</div>
<!-- /#page-wrapper -->
<script>
@ -99,17 +108,16 @@
$("#"+activePage).addClass("active");
$('[data-toggle="tooltip"]').tooltip();
table_track = $('#myTable').DataTable();
table_black = $('#myTable2').DataTable();
table_track = $('#myTable').DataTable({ "order": [[ 1, "dec" ]] });
table_track.on( 'draw.dt', function () {
perform_binding();
});
$("#followCredInput").keyup(function(event){
$("#seekInput").keyup(function(event){
if(event.keyCode == 13){
$("#followTermBtn").click();
$("#followCredInput").val("");
$("#seekInput").val("");
}
});
@ -193,25 +201,37 @@
$.get("{{ url_for('terms.cred_management_action') }}", data_to_send, function(data, status){
if(status == "success") {
var json = data;
console.log(json)
if(json.action == "add") {
} else if (json.action == "seek") {
var rep = json.data
var action_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"left\" title=\"Remove this term\" data-content=\"" + json.term + "\" data-section=\"followTerm\" data-action=\"delete\"><span class=\"glyphicon glyphicon-trash\"></span></button>"
table_track.clear().draw();
var rep = json.data;
var action_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"left\" title=\"Track this term\" data-content=\"" + json.term + "\" data-section=\"followTerm\" data-action=\"add\">";
if (rep.usr.length == 0) {
console.log('noData');
$( "#nodata" ).removeClass('alert-info').addClass('alert-danger');
$( "#nodata" ).text("No data");
$( "#nodata" ).fadeIn( "fast", function() {
setTimeout(function() {
$( "#nodata" ).fadeOut("fast");
}, 2000);
});
} else {
for(i=0; i < rep.length; i++) {
table_track.row.add( [ json.term, "xx/xx/xxxx", "xxxx", "xxxx", action_button ] ).draw( false );
console.log(json);
console.log(table_track);
for(i=0; i < rep.usr.length; i++) {
$( "#nodata" ).removeClass('alert-danger').addClass('alert-info');
$( "#nodata" ).text(curr_term);
$( "#nodata" ).fadeIn( "fast");
toAdd = "<span class=\"fa fa-eye\"></span></button><span data-toggle=\"modal\" data-target=\"#mymodal\" data-term=\""+rep.usr[i]+"\" ><button class=\"btn-link\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Show concerned paste(s)\"><span class=\"glyphicon glyphicon-info-sign\"></span></button></span>";
table_track.row.add( [
rep.usr[i],
rep.simil[i],
rep.numPaste[i],
toAdd+action_button ] ).draw( false );
}
//perform_binding();
perform_binding();
}
}
}
@ -220,7 +240,6 @@
}
</script>
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
</body>
</html>

View File

@ -2,7 +2,7 @@
<span class="caret"></span></a>
<ul class="dropdown-menu">
<li><a href="{{ url_for('terms.terms_management') }}"><i class="fa fa-gear "> </i> Terms managements</a></li>
<li><a href="{{ url_for('terms.credentials_tracker') }}"><i class="glyphicon glyphicon-screenshot"> </i> Credentials tracker</a></li>
<li><a href="{{ url_for('terms.credentials_tracker') }}"><i class="glyphicon glyphicon-screenshot"> </i> Credentials seeker</a></li>
<li><a href="{{ url_for('terms.terms_plot_top') }}"><i class="glyphicon glyphicon-fire"> </i> Terms plot top</a></li>
<li><a href="{{ url_for('terms.terms_plot_tool') }}"><i class="fa fa-wrench"> </i> Terms plot tool</a></li>
</ul>

View File

@ -20,7 +20,7 @@
<span class="caret"></span></a>
<ul class="dropdown-menu">
<li><a href="{{ url_for('terms.terms_management') }}"><i class="fa fa-gear "> </i> Terms managements</a></li>
<li><a href="{{ url_for('terms.credentials_tracker') }}"><i class="glyphicon glyphicon-screenshot"> </i> Credentials tracker</a></li>
<li><a href="{{ url_for('terms.credentials_tracker') }}"><i class="glyphicon glyphicon-screenshot"> </i> Credentials seeker</a></li>
<li><a href="{{ url_for('terms.terms_plot_top') }}"><i class="glyphicon glyphicon-fire"> </i> Terms plot top</a></li>
<li><a href="{{ url_for('terms.terms_plot_tool') }}"><i class="fa fa-wrench"> </i> Terms plot tool</a></li>
</ul>