From 9b8570e40bf21e541066f71924716d8d069f850b Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Mon, 17 Jul 2017 17:26:19 +0200 Subject: [PATCH 1/6] Dirty draft webpage credential_tracking --- var/www/modules/terms/Flask_terms.py | 19 ++ .../terms/templates/credentials_tracker.html | 224 ++++++++++++++++++ .../modules/terms/templates/header_terms.html | 1 + var/www/templates/header.html | 1 + 4 files changed, 245 insertions(+) create mode 100644 var/www/modules/terms/templates/credentials_tracker.html diff --git a/var/www/modules/terms/Flask_terms.py b/var/www/modules/terms/Flask_terms.py index 3df9bede..75362c31 100644 --- a/var/www/modules/terms/Flask_terms.py +++ b/var/www/modules/terms/Flask_terms.py @@ -348,5 +348,24 @@ def terms_plot_top_data(): return jsonify(to_return) +@terms.route("/credentials_tracker/") +def credentials_tracker(): + return render_template("credentials_tracker.html") + +@terms.route("/credentials_management_query_paste/") +def credentials_management_query_paste(): + cred = request.args.get('cred') + return 1 + +@terms.route("/credentials_management_action/", methods=['GET']) +def cred_management_action(): + cred = request.args.get('cred') + action = request.args.get('action') + return 1 + +@terms.route("/credentials_management_query/") +def cred_management_query(): + return 1 + # ========= REGISTRATION ========= app.register_blueprint(terms) diff --git a/var/www/modules/terms/templates/credentials_tracker.html b/var/www/modules/terms/templates/credentials_tracker.html new file mode 100644 index 00000000..9f04ebea --- /dev/null +++ b/var/www/modules/terms/templates/credentials_tracker.html @@ -0,0 +1,224 @@ + + + + + + + + Analysis Information Leak framework Dashboard + + + + + + + + + + + + + + + + + + + + + + + {% include 'navbar.html' %} + +
+
+
+

Credentials tracking

+
+ + +
+
+
+
+
+
+ Manage tracked credentials +
+
+ +
+ + + +
+ + + + + + + + + + + + + + + + {% for set in trackSet_list %} + + + + + + + + + + {% endfor %} + +
CredentialAdded dateDay occurenceWeek occurenceMonth occurence# tracked pasteAction
{{ set }}{{ trackSet_list_values[loop.index0][3] }}{{ trackSet_list_values[loop.index0][0] }}{{ trackSet_list_values[loop.index0][1] }}{{ trackSet_list_values[loop.index0][2] }}{{ trackSet_list_num_of_paste[loop.index0] }}

+ + +

+ +
+
+ +
+ +
+
+
+ +
+ + + + + + + + + + diff --git a/var/www/modules/terms/templates/header_terms.html b/var/www/modules/terms/templates/header_terms.html index d834bb31..e5970aaa 100644 --- a/var/www/modules/terms/templates/header_terms.html +++ b/var/www/modules/terms/templates/header_terms.html @@ -2,6 +2,7 @@ diff --git a/var/www/templates/header.html b/var/www/templates/header.html index 34682d07..8d189318 100644 --- a/var/www/templates/header.html +++ b/var/www/templates/header.html @@ -20,6 +20,7 @@ From 882ab88b00ed94758a00bb75a59e334a5378ae52 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Tue, 18 Jul 2017 16:57:15 +0200 Subject: [PATCH 2/6] Draft module + web link --- bin/Credential.py | 57 ++++++++++++++ bin/packages/config.cfg.sample | 5 ++ var/www/modules/Flask_config.py | 5 ++ var/www/modules/terms/Flask_terms.py | 75 ++++++++++++++++++- .../terms/templates/credentials_tracker.html | 54 ++++++------- 5 files changed, 168 insertions(+), 28 deletions(-) diff --git a/bin/Credential.py b/bin/Credential.py index ff8f8f97..583d0457 100755 --- a/bin/Credential.py +++ b/bin/Credential.py @@ -9,6 +9,17 @@ This module is consuming the Redis-list created by the Categ module. It apply credential regexes on paste content and warn if above a threshold. +It also split the username and store it into redis for searching purposes. + +Redis organization: + uniqNumForUsername: unique number attached to unique username + uniqNumForPath: unique number attached to unique path + AllCredentials: hashed set where keys are username and value are their uniq number + AllCredentialsRev: the opposite of AllCredentials, uniqNum -> username + AllPath: hashed set where keys are path and value are their uniq number + AllPathRev: the opposite of AllPath, uniqNum -> path + splitedCred -> uniq_num (set) + """ import time @@ -17,8 +28,19 @@ from packages import Paste from pubsublogger import publisher from Helper import Process import re +import redis from pyfaup.faup import Faup +#split username with spec. char or with upper case, distinguish start with upper +REGEX_CRED = "[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+" +REDIS_KEY_NUM_USERNAME = 'uniqNumForUsername' +REDIS_KEY_NUM_PATH = 'uniqNumForUsername' +REDIS_KEY_ALL_CRED_SET = 'AllCredentials' +REDIS_KEY_ALL_CRED_SET_REV = 'AllCredentialsRev' +REDIS_KEY_ALL_PATH_SET = 'AllPath' +REDIS_KEY_ALL_PATH_SET_REV = 'AllPathRev' +REDIS_KEY_MAP_CRED_TO_PATH = 'CredToPathMapping' + if __name__ == "__main__": publisher.port = 6380 publisher.channel = "Script" @@ -27,6 +49,10 @@ if __name__ == "__main__": publisher.info("Find credentials") faup = Faup() + server_cred = redis.StrictRedis( + host=p.config.get("Redis_Level_DB_TermCred", "host"), + port=p.config.get("Redis_Level_DB_TermCred", "port"), + db=p.config.get("Redis_Level_DB_TermCred", "db")) critical = 8 @@ -37,6 +63,7 @@ if __name__ == "__main__": message = p.get_from_set() if message is None: publisher.debug("Script Credential is Idling 10s") + print('sleeping 10s') time.sleep(10) continue @@ -44,6 +71,7 @@ if __name__ == "__main__": if count < 5: # Less than 5 matches from the top password list, false positive. + print("false positive:", count) continue paste = Paste.Paste(filepath) @@ -63,6 +91,7 @@ if __name__ == "__main__": print('\n '.join(creds)) + #num of creds above tresh, publish an alert if len(creds) > critical: print("========> Found more than 10 credentials in this file : {}".format(filepath)) publisher.warning(to_print) @@ -97,3 +126,31 @@ if __name__ == "__main__": print("=======> Probably on : {}".format(', '.join(sites_set))) else: publisher.info(to_print) + print('found {} credentials'.format(len(creds))) + + + #for searching credential in cred seeker + for cred in creds: + cred = cred.split('@')[0] + + #unique number attached to unique path + uniq_num_path = server_cred.incr(REDIS_KEY_ALL_PATH_SET) + print(REDIS_KEY_ALL_PATH_SET, {filepath: uniq_num_path}) + server_cred.hmset(REDIS_KEY_ALL_PATH_SET, {filepath: uniq_num_path}) + server_cred.hmset(REDIS_KEY_ALL_PATH_SET_REV, {uniq_num_path: filepath}) + + #unique number attached to unique username + uniq_num_cred = server_cred.hget(REDIS_KEY_ALL_CRED_SET, cred) + if uniq_num_cred is None: #cred do not exist, create new entries + uniq_num_cred = server_cred.incr(REDIS_KEY_NUM_USERNAME) + server_cred.hmset(REDIS_KEY_ALL_CRED_SET, {cred: uniq_num_cred}) + server_cred.hmset(REDIS_KEY_ALL_CRED_SET_REV, {uniq_num_cred: cred}) + + server_cred.hmset(REDIS_KEY_MAP_CRED_TO_PATH, {uniq_num_cred: uniq_num_path}) + + splitedCred = re.findall(REGEX_CRED, cred) + print(splitedCred) + for partCred in splitedCred: + server_cred.sadd(partCred, uniq_num_cred) + + diff --git a/bin/packages/config.cfg.sample b/bin/packages/config.cfg.sample index 0e91a993..2e2d246d 100644 --- a/bin/packages/config.cfg.sample +++ b/bin/packages/config.cfg.sample @@ -97,6 +97,11 @@ host = localhost port = 6382 db = 2 +[Redis_Level_DB_TermCred] +host = localhost +port = 6382 +db = 5 + [Redis_Level_DB] host = localhost port = 2016 diff --git a/var/www/modules/Flask_config.py b/var/www/modules/Flask_config.py index c15e4dca..161e6130 100644 --- a/var/www/modules/Flask_config.py +++ b/var/www/modules/Flask_config.py @@ -53,6 +53,11 @@ r_serv_term = redis.StrictRedis( port=cfg.getint("Redis_Level_DB_TermFreq", "port"), db=cfg.getint("Redis_Level_DB_TermFreq", "db")) +r_serv_cred = redis.StrictRedis( + host=cfg.get("Redis_Level_DB_TermCred", "host"), + port=cfg.getint("Redis_Level_DB_TermCred", "port"), + db=cfg.getint("Redis_Level_DB_TermCred", "db")) + r_serv_pasteName = redis.StrictRedis( host=cfg.get("Redis_Paste_Name", "host"), port=cfg.getint("Redis_Paste_Name", "port"), diff --git a/var/www/modules/terms/Flask_terms.py b/var/www/modules/terms/Flask_terms.py index 75362c31..535ddafe 100644 --- a/var/www/modules/terms/Flask_terms.py +++ b/var/www/modules/terms/Flask_terms.py @@ -11,6 +11,7 @@ import flask from flask import Flask, render_template, jsonify, request, Blueprint import re import Paste +from pprint import pprint # ============ VARIABLES ============ import Flask_config @@ -18,6 +19,7 @@ import Flask_config app = Flask_config.app cfg = Flask_config.cfg r_serv_term = Flask_config.r_serv_term +r_serv_cred = Flask_config.r_serv_cred terms = Blueprint('terms', __name__, template_folder='templates') @@ -357,11 +359,80 @@ def credentials_management_query_paste(): cred = request.args.get('cred') return 1 +def mixUserName(supplied): + #e.g.: John Smith + terms = supplied.split()[:2] + usernames = [] + if len(terms) == 1: + terms.append(' ') + + #john, smith, John, Smith, JOHN, SMITH + usernames += [terms[0].lower()] + usernames += [terms[1].lower()] + usernames += [terms[0][0].upper() + terms[0][1:].lower()] + usernames += [terms[1][0].upper() + terms[1][1:].lower()] + usernames += [terms[0].upper()] + usernames += [terms[1].upper()] + + #johnsmith, smithjohn, JOHNsmith, johnSMITH, SMITHjohn, smithJOHN + usernames += [(terms[0].lower() + terms[1].lower()).strip()] + usernames += [(terms[1].lower() + terms[0].lower()).strip()] + usernames += [(terms[0].upper() + terms[1].lower()).strip()] + usernames += [(terms[0].lower() + terms[1].upper()).strip()] + usernames += [(terms[1].upper() + terms[0].lower()).strip()] + usernames += [(terms[1].lower() + terms[0].upper()).strip()] + #Jsmith, JSmith, jsmith, jSmith, johnS, Js, JohnSmith, Johnsmith, johnSmith + usernames += [(terms[0][0].upper() + terms[1][0].lower() + terms[1][1:].lower()).strip()] + usernames += [(terms[0][0].upper() + terms[1][0].upper() + terms[1][1:].lower()).strip()] + usernames += [(terms[0][0].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()] + usernames += [(terms[0][0].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()] + usernames += [(terms[0].lower() + terms[1][0].upper()).strip()] + usernames += [(terms[0].upper() + terms[1][0].lower()).strip()] + usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()] + usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()] + usernames += [(terms[0][0].lower() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()] + + return usernames + + + @terms.route("/credentials_management_action/", methods=['GET']) def cred_management_action(): - cred = request.args.get('cred') + REGEX_CRED = '[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+' + REDIS_KEY_NUM_USERNAME = 'uniqNumForUsername' + REDIS_KEY_NUM_PATH = 'uniqNumForUsername' + REDIS_KEY_ALL_CRED_SET = 'AllCredentials' + REDIS_KEY_ALL_CRED_SET_REV = 'AllCredentialsRev' + REDIS_KEY_ALL_PATH_SET = 'AllPath' + REDIS_KEY_ALL_PATH_SET_REV = 'AllPath' + REDIS_KEY_MAP_CRED_TO_PATH = 'CredToPathMapping' + + supplied = request.args.get('term') action = request.args.get('action') - return 1 + section = request.args.get('section') + + #splitedCred = re.findall(REGEX_CRED, cred) + uniq_num_set = set() + if action == "seek": + possibilities = mixUserName(supplied) + for poss in possibilities: + for num in r_serv_cred.smembers(poss): + uniq_num_set.add(num) + + data = {'usr': [], 'path': []} + for Unum in uniq_num_set: + data['usr'].append(r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET_REV, Unum)) + data['path'].append(r_serv_cred.hget(REDIS_KEY_MAP_CRED_TO_PATH, Unum)) + + pprint(data) + to_return = {} + to_return["section"] = section + to_return["action"] = action + to_return["term"] = supplied + to_return["data"] = data + + return jsonify(to_return) + @terms.route("/credentials_management_query/") def cred_management_query(): diff --git a/var/www/modules/terms/templates/credentials_tracker.html b/var/www/modules/terms/templates/credentials_tracker.html index 9f04ebea..08fe2fec 100644 --- a/var/www/modules/terms/templates/credentials_tracker.html +++ b/var/www/modules/terms/templates/credentials_tracker.html @@ -33,7 +33,7 @@
-

Credentials tracking

+

Credential seeker

@@ -43,24 +43,23 @@
- Manage tracked credentials + Credential seeker
- - + + +
- - - - + + @@ -70,14 +69,12 @@ {% for set in trackSet_list %} - - {% endfor %} @@ -184,32 +181,37 @@ var curr_action = $(this).attr('data-action'); if (curr_action == "add") { var curr_term = $('#'+curr_section+'Input').val(); + } else if (curr_action == "seek") { + var curr_term = $('#'+curr_section+'Input').val(); } else { var curr_term = $(this).attr('data-content'); } var data_to_send = { section: curr_section, action:curr_action, term: curr_term}; if (curr_term != "") { - console.log(data_to_send); + //console.log(data_to_send); $.get("{{ url_for('terms.cred_management_action') }}", data_to_send, function(data, status){ if(status == "success") { var json = data; + console.log(json) - if(json.section == "followTerm") { - if(json.action == "add") { - // query data - $.get("{{ url_for('terms.cred_management_query') }}", { term: json.term, section: json.section }, function(data2, status){ - var action_button = "" - table_track.row.add( [ json.term, data2[3], data2[0], data2[1], data2[2], 0, action_button ] ).draw( false ); - perform_binding(); + if(json.action == "add") { + + } else if (json.action == "seek") { + var rep = json.data + var action_button = "" + if (rep.usr.length == 0) { + console.log('noData'); + $( "#nodata" ).fadeIn( "fast", function() { + setTimeout(function() { + $( "#nodata" ).fadeOut("fast"); + }, 2000); }); - } else if (json.action == "delete") { - // Find indexes of row which have the term in the first column - var index = table_track.rows().eq( 0 ).filter( function (rowIdx) { - console.log(table_track.cell( rowIdx, 0 ).data()) - return table_track.cell( rowIdx, 0 ).data() === json.term; - } ); - table_track.rows(index).remove().draw( false ); + } else { + for(i=0; i < rep.length; i++) { + table_track.row.add( [ json.term, "xx/xx/xxxx", "xxxx", "xxxx", action_button ] ).draw( false ); + } + //perform_binding(); } } } From ef741f85c07e179f23890a371fec8ef638743709 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Wed, 19 Jul 2017 11:52:06 +0200 Subject: [PATCH 3/6] Added similarity checks + started link webInterface --- bin/Credential.py | 8 +- pip_packages_requirement.txt | 1 + var/www/modules/terms/Flask_terms.py | 92 ++++++----- .../terms/templates/credentials_tracker.html | 153 ++++++++++-------- .../modules/terms/templates/header_terms.html | 2 +- var/www/templates/header.html | 2 +- 6 files changed, 146 insertions(+), 112 deletions(-) diff --git a/bin/Credential.py b/bin/Credential.py index 583d0457..baab72f1 100755 --- a/bin/Credential.py +++ b/bin/Credential.py @@ -40,6 +40,7 @@ REDIS_KEY_ALL_CRED_SET_REV = 'AllCredentialsRev' REDIS_KEY_ALL_PATH_SET = 'AllPath' REDIS_KEY_ALL_PATH_SET_REV = 'AllPathRev' REDIS_KEY_MAP_CRED_TO_PATH = 'CredToPathMapping' +MINIMUMSIZETHRESHOLD = 3 if __name__ == "__main__": publisher.port = 6380 @@ -53,7 +54,7 @@ if __name__ == "__main__": host=p.config.get("Redis_Level_DB_TermCred", "host"), port=p.config.get("Redis_Level_DB_TermCred", "port"), db=p.config.get("Redis_Level_DB_TermCred", "db")) - +Credential critical = 8 regex_web = "((?:https?:\/\/)[-_0-9a-zA-Z]+\.[0-9a-zA-Z]+)" @@ -134,7 +135,7 @@ if __name__ == "__main__": cred = cred.split('@')[0] #unique number attached to unique path - uniq_num_path = server_cred.incr(REDIS_KEY_ALL_PATH_SET) + uniq_num_path = server_cred.incr(REDIS_KEY_NUM_PATH) print(REDIS_KEY_ALL_PATH_SET, {filepath: uniq_num_path}) server_cred.hmset(REDIS_KEY_ALL_PATH_SET, {filepath: uniq_num_path}) server_cred.hmset(REDIS_KEY_ALL_PATH_SET_REV, {uniq_num_path: filepath}) @@ -151,6 +152,7 @@ if __name__ == "__main__": splitedCred = re.findall(REGEX_CRED, cred) print(splitedCred) for partCred in splitedCred: - server_cred.sadd(partCred, uniq_num_cred) + if len(partCred) > MINIMUMSIZETHRESHOLD: + server_cred.sadd(partCred, uniq_num_cred) diff --git a/pip_packages_requirement.txt b/pip_packages_requirement.txt index 8bffb7b5..d6dd108d 100644 --- a/pip_packages_requirement.txt +++ b/pip_packages_requirement.txt @@ -21,6 +21,7 @@ nltk crcmod mmh3 ssdeep +python-Levenshtein #Others python-magic diff --git a/var/www/modules/terms/Flask_terms.py b/var/www/modules/terms/Flask_terms.py index 535ddafe..66564f93 100644 --- a/var/www/modules/terms/Flask_terms.py +++ b/var/www/modules/terms/Flask_terms.py @@ -12,6 +12,7 @@ from flask import Flask, render_template, jsonify, request, Blueprint import re import Paste from pprint import pprint +import Levenshtein # ============ VARIABLES ============ import Flask_config @@ -54,6 +55,41 @@ def Term_getValueOverRange(word, startDate, num_day, per_paste=""): passed_days += 1 return to_return +def mixUserName(supplied): + #e.g.: John Smith + terms = supplied.split()[:2] + usernames = [] + if len(terms) == 1: + terms.append(' ') + + #john, smith, John, Smith, JOHN, SMITH + usernames += [terms[0].lower()] + usernames += [terms[1].lower()] + usernames += [terms[0][0].upper() + terms[0][1:].lower()] + usernames += [terms[1][0].upper() + terms[1][1:].lower()] + usernames += [terms[0].upper()] + usernames += [terms[1].upper()] + + #johnsmith, smithjohn, JOHNsmith, johnSMITH, SMITHjohn, smithJOHN + usernames += [(terms[0].lower() + terms[1].lower()).strip()] + usernames += [(terms[1].lower() + terms[0].lower()).strip()] + usernames += [(terms[0].upper() + terms[1].lower()).strip()] + usernames += [(terms[0].lower() + terms[1].upper()).strip()] + usernames += [(terms[1].upper() + terms[0].lower()).strip()] + usernames += [(terms[1].lower() + terms[0].upper()).strip()] + #Jsmith, JSmith, jsmith, jSmith, johnS, Js, JohnSmith, Johnsmith, johnSmith + usernames += [(terms[0][0].upper() + terms[1][0].lower() + terms[1][1:].lower()).strip()] + usernames += [(terms[0][0].upper() + terms[1][0].upper() + terms[1][1:].lower()).strip()] + usernames += [(terms[0][0].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()] + usernames += [(terms[0][0].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()] + usernames += [(terms[0].lower() + terms[1][0].upper()).strip()] + usernames += [(terms[0].upper() + terms[1][0].lower()).strip()] + usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()] + usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()] + usernames += [(terms[0][0].lower() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()] + + return usernames + # ============ ROUTES ============ @@ -359,41 +395,7 @@ def credentials_management_query_paste(): cred = request.args.get('cred') return 1 -def mixUserName(supplied): - #e.g.: John Smith - terms = supplied.split()[:2] - usernames = [] - if len(terms) == 1: - terms.append(' ') - - #john, smith, John, Smith, JOHN, SMITH - usernames += [terms[0].lower()] - usernames += [terms[1].lower()] - usernames += [terms[0][0].upper() + terms[0][1:].lower()] - usernames += [terms[1][0].upper() + terms[1][1:].lower()] - usernames += [terms[0].upper()] - usernames += [terms[1].upper()] - - #johnsmith, smithjohn, JOHNsmith, johnSMITH, SMITHjohn, smithJOHN - usernames += [(terms[0].lower() + terms[1].lower()).strip()] - usernames += [(terms[1].lower() + terms[0].lower()).strip()] - usernames += [(terms[0].upper() + terms[1].lower()).strip()] - usernames += [(terms[0].lower() + terms[1].upper()).strip()] - usernames += [(terms[1].upper() + terms[0].lower()).strip()] - usernames += [(terms[1].lower() + terms[0].upper()).strip()] - #Jsmith, JSmith, jsmith, jSmith, johnS, Js, JohnSmith, Johnsmith, johnSmith - usernames += [(terms[0][0].upper() + terms[1][0].lower() + terms[1][1:].lower()).strip()] - usernames += [(terms[0][0].upper() + terms[1][0].upper() + terms[1][1:].lower()).strip()] - usernames += [(terms[0][0].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()] - usernames += [(terms[0][0].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()] - usernames += [(terms[0].lower() + terms[1][0].upper()).strip()] - usernames += [(terms[0].upper() + terms[1][0].lower()).strip()] - usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()] - usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()] - usernames += [(terms[0][0].lower() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()] - - return usernames - + @terms.route("/credentials_management_action/", methods=['GET']) @@ -407,7 +409,7 @@ def cred_management_action(): REDIS_KEY_ALL_PATH_SET_REV = 'AllPath' REDIS_KEY_MAP_CRED_TO_PATH = 'CredToPathMapping' - supplied = request.args.get('term') + supplied = request.args.get('term').encode('utf-8') action = request.args.get('action') section = request.args.get('section') @@ -419,12 +421,22 @@ def cred_management_action(): for num in r_serv_cred.smembers(poss): uniq_num_set.add(num) - data = {'usr': [], 'path': []} + data = {'usr': [], 'path': [], 'numPaste': [], 'simil': []} for Unum in uniq_num_set: - data['usr'].append(r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET_REV, Unum)) - data['path'].append(r_serv_cred.hget(REDIS_KEY_MAP_CRED_TO_PATH, Unum)) + username = r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET_REV, Unum) + + # Calculate Levenshtein distance, ignore negative ratio + levenDist = float(Levenshtein.distance(supplied, username)) + levenRatio = levenDist / float(len(supplied)) + levenRatioStr = "{:.1%}".format(1.0 - levenRatio) + if levenRatio >= 1.0: + continue + + data['usr'].append(username) + data['path'].append(r_serv_cred.hget(REDIS_KEY_MAP_CRED_TO_PATH, Unum)) + data['numPaste'].append(len(uniq_num_set)) + data['simil'].append(levenRatioStr) - pprint(data) to_return = {} to_return["section"] = section to_return["action"] = action diff --git a/var/www/modules/terms/templates/credentials_tracker.html b/var/www/modules/terms/templates/credentials_tracker.html index 08fe2fec..1293aae1 100644 --- a/var/www/modules/terms/templates/credentials_tracker.html +++ b/var/www/modules/terms/templates/credentials_tracker.html @@ -1,40 +1,62 @@ - - - - - Analysis Information Leak framework Dashboard - - - - - - - - - - - - - - - - - - - - - - - {% include 'navbar.html' %} - -
-
-
-

Credential seeker

-
+ + + + + Analysis Information Leak framework Dashboard + + + + + + + + + + + + + + + + + + + + + + + + + + {% include 'navbar.html' %} + +
+
+
+

Credential seeker

+
@@ -48,36 +70,22 @@
- + - +
CredentialAdded dateDay occurenceWeek occurenceMonth occurenceDate# line in the paste # tracked paste Action
{{ set }}{{ trackSet_list_values[loop.index0][3] }} {{ trackSet_list_values[loop.index0][0] }} {{ trackSet_list_values[loop.index0][1] }}{{ trackSet_list_values[loop.index0][2] }} {{ trackSet_list_num_of_paste[loop.index0] }}

- +

- - - - + + + - - {% for set in trackSet_list %} - - - - - - - - {% endfor %}
CredentialDate# line in the paste# tracked pasteUsernameSimilarity# concerned paste(s) Action
{{ set }}{{ trackSet_list_values[loop.index0][0] }}{{ trackSet_list_values[loop.index0][1] }}{{ trackSet_list_num_of_paste[loop.index0] }}

- - -

@@ -88,9 +96,10 @@
-
- -
+ +
+ +
- diff --git a/var/www/modules/terms/templates/header_terms.html b/var/www/modules/terms/templates/header_terms.html index e5970aaa..8fa38bc5 100644 --- a/var/www/modules/terms/templates/header_terms.html +++ b/var/www/modules/terms/templates/header_terms.html @@ -2,7 +2,7 @@ diff --git a/var/www/templates/header.html b/var/www/templates/header.html index 8d189318..1df983a0 100644 --- a/var/www/templates/header.html +++ b/var/www/templates/header.html @@ -20,7 +20,7 @@ From 05417f405c8961b20635709fa8ac91d09712802c Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Thu, 20 Jul 2017 10:04:30 +0200 Subject: [PATCH 4/6] Finished basic link web interface + solved some bugs --- bin/Credential.py | 9 +- var/www/modules/terms/Flask_terms.py | 108 ++++++++++++++---- .../terms/templates/credentials_tracker.html | 55 +++++---- 3 files changed, 121 insertions(+), 51 deletions(-) diff --git a/bin/Credential.py b/bin/Credential.py index baab72f1..db2125cd 100755 --- a/bin/Credential.py +++ b/bin/Credential.py @@ -18,7 +18,7 @@ Redis organization: AllCredentialsRev: the opposite of AllCredentials, uniqNum -> username AllPath: hashed set where keys are path and value are their uniq number AllPathRev: the opposite of AllPath, uniqNum -> path - splitedCred -> uniq_num (set) + CredToPathMapping_uniqNumForUsername -> (set) -> uniqNumForPath """ @@ -54,7 +54,7 @@ if __name__ == "__main__": host=p.config.get("Redis_Level_DB_TermCred", "host"), port=p.config.get("Redis_Level_DB_TermCred", "port"), db=p.config.get("Redis_Level_DB_TermCred", "db")) -Credential + critical = 8 regex_web = "((?:https?:\/\/)[-_0-9a-zA-Z]+\.[0-9a-zA-Z]+)" @@ -136,7 +136,6 @@ Credential #unique number attached to unique path uniq_num_path = server_cred.incr(REDIS_KEY_NUM_PATH) - print(REDIS_KEY_ALL_PATH_SET, {filepath: uniq_num_path}) server_cred.hmset(REDIS_KEY_ALL_PATH_SET, {filepath: uniq_num_path}) server_cred.hmset(REDIS_KEY_ALL_PATH_SET_REV, {uniq_num_path: filepath}) @@ -147,10 +146,10 @@ Credential server_cred.hmset(REDIS_KEY_ALL_CRED_SET, {cred: uniq_num_cred}) server_cred.hmset(REDIS_KEY_ALL_CRED_SET_REV, {uniq_num_cred: cred}) - server_cred.hmset(REDIS_KEY_MAP_CRED_TO_PATH, {uniq_num_cred: uniq_num_path}) + #server_cred.hmset(REDIS_KEY_MAP_CRED_TO_PATH, {uniq_num_cred: uniq_num_path}) + server_cred.sadd(REDIS_KEY_MAP_CRED_TO_PATH+'_'+str(uniq_num_cred), uniq_num_path) splitedCred = re.findall(REGEX_CRED, cred) - print(splitedCred) for partCred in splitedCred: if len(partCred) > MINIMUMSIZETHRESHOLD: server_cred.sadd(partCred, uniq_num_cred) diff --git a/var/www/modules/terms/Flask_terms.py b/var/www/modules/terms/Flask_terms.py index 66564f93..a0bcb8fe 100644 --- a/var/www/modules/terms/Flask_terms.py +++ b/var/www/modules/terms/Flask_terms.py @@ -24,6 +24,7 @@ r_serv_cred = Flask_config.r_serv_cred terms = Blueprint('terms', __name__, template_folder='templates') +'''TERM''' DEFAULT_MATCH_PERCENT = 50 #tracked @@ -39,6 +40,19 @@ TrackedRegexDate_Name = "TrackedRegexDate" TrackedSetSet_Name = "TrackedSetSet" TrackedSetDate_Name = "TrackedSetDate" + +'''CRED''' +REGEX_CRED = '[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+' +REDIS_KEY_NUM_USERNAME = 'uniqNumForUsername' +REDIS_KEY_NUM_PATH = 'uniqNumForUsername' +REDIS_KEY_ALL_CRED_SET = 'AllCredentials' +REDIS_KEY_ALL_CRED_SET_REV = 'AllCredentialsRev' +REDIS_KEY_ALL_PATH_SET = 'AllPath' +REDIS_KEY_ALL_PATH_SET_REV = 'AllPathRev' +REDIS_KEY_MAP_CRED_TO_PATH = 'CredToPathMapping' + + + # ============ FUNCTIONS ============ def Term_getValueOverRange(word, startDate, num_day, per_paste=""): @@ -55,7 +69,7 @@ def Term_getValueOverRange(word, startDate, num_day, per_paste=""): passed_days += 1 return to_return -def mixUserName(supplied): +def mixUserName(supplied, extensive=False): #e.g.: John Smith terms = supplied.split()[:2] usernames = [] @@ -88,7 +102,20 @@ def mixUserName(supplied): usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()] usernames += [(terms[0][0].lower() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()] - return usernames + if not extensive: + return usernames + + mixedSupplied = supplied.replace(' ','') + minWindow = 3 if len(mixedSupplied)/2 < 4 else len(mixedSupplied)/2 + for winSize in range(3,len(mixedSupplied)): + for startIndex in range(0, len(mixedSupplied)-winSize): + usernames += [mixedSupplied[startIndex:startIndex+winSize]] + + filtered_usernames = [] + for usr in usernames: + if len(usr) > 2: + filtered_usernames.append(usr) + return filtered_usernames # ============ ROUTES ============ @@ -305,6 +332,7 @@ def terms_plot_tool(): @terms.route("/terms_plot_tool_data/") def terms_plot_tool_data(): + oneDay = 60*60*24 range_start = datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_start')))) if request.args.get('range_start') is not None else 0; range_start = range_start.replace(hour=0, minute=0, second=0, microsecond=0) @@ -323,6 +351,7 @@ def terms_plot_tool_data(): if term is None: return "None" + else: value_range = [] for timestamp in range(range_start, range_end+oneDay, oneDay): @@ -335,6 +364,7 @@ def terms_plot_tool_data(): @terms.route("/terms_plot_top/") def terms_plot_top(): + per_paste = request.args.get('per_paste') per_paste = per_paste if per_paste is not None else 1 return render_template("terms_plot_top.html", per_paste=per_paste) @@ -347,6 +377,7 @@ def terms_plot_top_data(): today = today.replace(hour=0, minute=0, second=0, microsecond=0) today_timestamp = calendar.timegm(today.timetuple()) + per_paste = request.args.get('per_paste') if per_paste == "1" or per_paste is None: per_paste = "per_paste_" @@ -390,51 +421,84 @@ def terms_plot_top_data(): def credentials_tracker(): return render_template("credentials_tracker.html") -@terms.route("/credentials_management_query_paste/") +@terms.route("/credentials_management_query_paste/", methods=['GET', 'POST']) def credentials_management_query_paste(): cred = request.args.get('cred') - return 1 + allPath = request.json['allPath'] - + paste_info = [] + for pathNum in allPath: + path = r_serv_cred.hget(REDIS_KEY_ALL_PATH_SET_REV, pathNum) + paste = Paste.Paste(path) + p_date = str(paste._get_p_date()) + p_date = p_date[6:]+'/'+p_date[4:6]+'/'+p_date[0:4] + p_source = paste.p_source + p_encoding = paste._get_p_encoding() + p_size = paste.p_size + p_mime = paste.p_mime + p_lineinfo = paste.get_lines_info() + p_content = paste.get_p_content().decode('utf-8', 'ignore') + if p_content != 0: + p_content = p_content[0:400] + paste_info.append({"path": path, "date": p_date, "source": p_source, "encoding": p_encoding, "size": p_size, "mime": p_mime, "lineinfo": p_lineinfo, "content": p_content}) + return jsonify(paste_info) @terms.route("/credentials_management_action/", methods=['GET']) def cred_management_action(): - REGEX_CRED = '[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+' - REDIS_KEY_NUM_USERNAME = 'uniqNumForUsername' - REDIS_KEY_NUM_PATH = 'uniqNumForUsername' - REDIS_KEY_ALL_CRED_SET = 'AllCredentials' - REDIS_KEY_ALL_CRED_SET_REV = 'AllCredentialsRev' - REDIS_KEY_ALL_PATH_SET = 'AllPath' - REDIS_KEY_ALL_PATH_SET_REV = 'AllPath' - REDIS_KEY_MAP_CRED_TO_PATH = 'CredToPathMapping' supplied = request.args.get('term').encode('utf-8') action = request.args.get('action') section = request.args.get('section') + extensive = request.args.get('extensive') + extensive = True if extensive == "true" else False - #splitedCred = re.findall(REGEX_CRED, cred) + if extensive: + #collectDico + AllUsernameInRedis = r_serv_cred.hgetall(REDIS_KEY_ALL_CRED_SET).keys() uniq_num_set = set() if action == "seek": - possibilities = mixUserName(supplied) + possibilities = mixUserName(supplied, extensive) for poss in possibilities: + num = r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET, poss) + if num is not None: + uniq_num_set.add(num) for num in r_serv_cred.smembers(poss): uniq_num_set.add(num) + #Extensive /!\ + if extensive: + for tempUsername in AllUsernameInRedis: + for poss in possibilities: + if poss in tempUsername: + num = r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET, tempUsername) + if num is not None: + uniq_num_set.add(num) + for num in r_serv_cred.smembers(tempUsername): + uniq_num_set.add(num) data = {'usr': [], 'path': [], 'numPaste': [], 'simil': []} for Unum in uniq_num_set: + levenRatio = 2.0 username = r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET_REV, Unum) # Calculate Levenshtein distance, ignore negative ratio - levenDist = float(Levenshtein.distance(supplied, username)) - levenRatio = levenDist / float(len(supplied)) - levenRatioStr = "{:.1%}".format(1.0 - levenRatio) - if levenRatio >= 1.0: - continue + supp_splitted = supplied.split() + supp_mixed = supplied.replace(' ','') + supp_splitted.append(supp_mixed) + for indiv_supplied in supp_splitted: + #levenDist = float(Levenshtein.distance(indiv_supplied, username)) + #levenRatio = levenDist / float(len(indiv_supplied)) if levenRatio > levenDist / float(len(indiv_supplied)) else levenRatio + #levenRatio = levenRatio if levenRatio < 1.0 else 1.0 + levenRatio = float(Levenshtein.ratio(indiv_supplied, username)) + levenRatioStr = "{:.1%}".format(levenRatio) + #levenRatioStr = "{:.1%}".format(1.0 - levenRatio) + #if levenRatio >= 1.0: + # continue data['usr'].append(username) - data['path'].append(r_serv_cred.hget(REDIS_KEY_MAP_CRED_TO_PATH, Unum)) - data['numPaste'].append(len(uniq_num_set)) + allPathNum = list(r_serv_cred.smembers(REDIS_KEY_MAP_CRED_TO_PATH+'_'+Unum)) + data['path'].append(allPathNum) + data['numPaste'].append(len(allPathNum)) data['simil'].append(levenRatioStr) to_return = {} diff --git a/var/www/modules/terms/templates/credentials_tracker.html b/var/www/modules/terms/templates/credentials_tracker.html index 1293aae1..e95e7db2 100644 --- a/var/www/modules/terms/templates/credentials_tracker.html +++ b/var/www/modules/terms/templates/credentials_tracker.html @@ -69,6 +69,7 @@
+
@@ -123,13 +124,33 @@ perform_binding(); + $("#mymodal").on('hidden.bs.modal', function () { + $("#mymodalbody").html("

Loading paste information...

"); + var loading_gif = ""; + $("#mymodalbody").append(loading_gif); // Show the loading GIF + }); + }); + + + - -