From ef741f85c07e179f23890a371fec8ef638743709 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Wed, 19 Jul 2017 11:52:06 +0200 Subject: [PATCH] Added similarity checks + started link webInterface --- bin/Credential.py | 8 +- pip_packages_requirement.txt | 1 + var/www/modules/terms/Flask_terms.py | 92 ++++++----- .../terms/templates/credentials_tracker.html | 153 ++++++++++-------- .../modules/terms/templates/header_terms.html | 2 +- var/www/templates/header.html | 2 +- 6 files changed, 146 insertions(+), 112 deletions(-) diff --git a/bin/Credential.py b/bin/Credential.py index 583d0457..baab72f1 100755 --- a/bin/Credential.py +++ b/bin/Credential.py @@ -40,6 +40,7 @@ REDIS_KEY_ALL_CRED_SET_REV = 'AllCredentialsRev' REDIS_KEY_ALL_PATH_SET = 'AllPath' REDIS_KEY_ALL_PATH_SET_REV = 'AllPathRev' REDIS_KEY_MAP_CRED_TO_PATH = 'CredToPathMapping' +MINIMUMSIZETHRESHOLD = 3 if __name__ == "__main__": publisher.port = 6380 @@ -53,7 +54,7 @@ if __name__ == "__main__": host=p.config.get("Redis_Level_DB_TermCred", "host"), port=p.config.get("Redis_Level_DB_TermCred", "port"), db=p.config.get("Redis_Level_DB_TermCred", "db")) - +Credential critical = 8 regex_web = "((?:https?:\/\/)[-_0-9a-zA-Z]+\.[0-9a-zA-Z]+)" @@ -134,7 +135,7 @@ if __name__ == "__main__": cred = cred.split('@')[0] #unique number attached to unique path - uniq_num_path = server_cred.incr(REDIS_KEY_ALL_PATH_SET) + uniq_num_path = server_cred.incr(REDIS_KEY_NUM_PATH) print(REDIS_KEY_ALL_PATH_SET, {filepath: uniq_num_path}) server_cred.hmset(REDIS_KEY_ALL_PATH_SET, {filepath: uniq_num_path}) server_cred.hmset(REDIS_KEY_ALL_PATH_SET_REV, {uniq_num_path: filepath}) @@ -151,6 +152,7 @@ if __name__ == "__main__": splitedCred = re.findall(REGEX_CRED, cred) print(splitedCred) for partCred in splitedCred: - server_cred.sadd(partCred, uniq_num_cred) + if len(partCred) > MINIMUMSIZETHRESHOLD: + server_cred.sadd(partCred, uniq_num_cred) diff --git a/pip_packages_requirement.txt b/pip_packages_requirement.txt index 8bffb7b5..d6dd108d 100644 --- a/pip_packages_requirement.txt +++ b/pip_packages_requirement.txt @@ -21,6 +21,7 @@ nltk crcmod mmh3 ssdeep +python-Levenshtein #Others python-magic diff --git a/var/www/modules/terms/Flask_terms.py b/var/www/modules/terms/Flask_terms.py index 535ddafe..66564f93 100644 --- a/var/www/modules/terms/Flask_terms.py +++ b/var/www/modules/terms/Flask_terms.py @@ -12,6 +12,7 @@ from flask import Flask, render_template, jsonify, request, Blueprint import re import Paste from pprint import pprint +import Levenshtein # ============ VARIABLES ============ import Flask_config @@ -54,6 +55,41 @@ def Term_getValueOverRange(word, startDate, num_day, per_paste=""): passed_days += 1 return to_return +def mixUserName(supplied): + #e.g.: John Smith + terms = supplied.split()[:2] + usernames = [] + if len(terms) == 1: + terms.append(' ') + + #john, smith, John, Smith, JOHN, SMITH + usernames += [terms[0].lower()] + usernames += [terms[1].lower()] + usernames += [terms[0][0].upper() + terms[0][1:].lower()] + usernames += [terms[1][0].upper() + terms[1][1:].lower()] + usernames += [terms[0].upper()] + usernames += [terms[1].upper()] + + #johnsmith, smithjohn, JOHNsmith, johnSMITH, SMITHjohn, smithJOHN + usernames += [(terms[0].lower() + terms[1].lower()).strip()] + usernames += [(terms[1].lower() + terms[0].lower()).strip()] + usernames += [(terms[0].upper() + terms[1].lower()).strip()] + usernames += [(terms[0].lower() + terms[1].upper()).strip()] + usernames += [(terms[1].upper() + terms[0].lower()).strip()] + usernames += [(terms[1].lower() + terms[0].upper()).strip()] + #Jsmith, JSmith, jsmith, jSmith, johnS, Js, JohnSmith, Johnsmith, johnSmith + usernames += [(terms[0][0].upper() + terms[1][0].lower() + terms[1][1:].lower()).strip()] + usernames += [(terms[0][0].upper() + terms[1][0].upper() + terms[1][1:].lower()).strip()] + usernames += [(terms[0][0].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()] + usernames += [(terms[0][0].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()] + usernames += [(terms[0].lower() + terms[1][0].upper()).strip()] + usernames += [(terms[0].upper() + terms[1][0].lower()).strip()] + usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()] + usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()] + usernames += [(terms[0][0].lower() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()] + + return usernames + # ============ ROUTES ============ @@ -359,41 +395,7 @@ def credentials_management_query_paste(): cred = request.args.get('cred') return 1 -def mixUserName(supplied): - #e.g.: John Smith - terms = supplied.split()[:2] - usernames = [] - if len(terms) == 1: - terms.append(' ') - - #john, smith, John, Smith, JOHN, SMITH - usernames += [terms[0].lower()] - usernames += [terms[1].lower()] - usernames += [terms[0][0].upper() + terms[0][1:].lower()] - usernames += [terms[1][0].upper() + terms[1][1:].lower()] - usernames += [terms[0].upper()] - usernames += [terms[1].upper()] - - #johnsmith, smithjohn, JOHNsmith, johnSMITH, SMITHjohn, smithJOHN - usernames += [(terms[0].lower() + terms[1].lower()).strip()] - usernames += [(terms[1].lower() + terms[0].lower()).strip()] - usernames += [(terms[0].upper() + terms[1].lower()).strip()] - usernames += [(terms[0].lower() + terms[1].upper()).strip()] - usernames += [(terms[1].upper() + terms[0].lower()).strip()] - usernames += [(terms[1].lower() + terms[0].upper()).strip()] - #Jsmith, JSmith, jsmith, jSmith, johnS, Js, JohnSmith, Johnsmith, johnSmith - usernames += [(terms[0][0].upper() + terms[1][0].lower() + terms[1][1:].lower()).strip()] - usernames += [(terms[0][0].upper() + terms[1][0].upper() + terms[1][1:].lower()).strip()] - usernames += [(terms[0][0].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()] - usernames += [(terms[0][0].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()] - usernames += [(terms[0].lower() + terms[1][0].upper()).strip()] - usernames += [(terms[0].upper() + terms[1][0].lower()).strip()] - usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()] - usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()] - usernames += [(terms[0][0].lower() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()] - - return usernames - + @terms.route("/credentials_management_action/", methods=['GET']) @@ -407,7 +409,7 @@ def cred_management_action(): REDIS_KEY_ALL_PATH_SET_REV = 'AllPath' REDIS_KEY_MAP_CRED_TO_PATH = 'CredToPathMapping' - supplied = request.args.get('term') + supplied = request.args.get('term').encode('utf-8') action = request.args.get('action') section = request.args.get('section') @@ -419,12 +421,22 @@ def cred_management_action(): for num in r_serv_cred.smembers(poss): uniq_num_set.add(num) - data = {'usr': [], 'path': []} + data = {'usr': [], 'path': [], 'numPaste': [], 'simil': []} for Unum in uniq_num_set: - data['usr'].append(r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET_REV, Unum)) - data['path'].append(r_serv_cred.hget(REDIS_KEY_MAP_CRED_TO_PATH, Unum)) + username = r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET_REV, Unum) + + # Calculate Levenshtein distance, ignore negative ratio + levenDist = float(Levenshtein.distance(supplied, username)) + levenRatio = levenDist / float(len(supplied)) + levenRatioStr = "{:.1%}".format(1.0 - levenRatio) + if levenRatio >= 1.0: + continue + + data['usr'].append(username) + data['path'].append(r_serv_cred.hget(REDIS_KEY_MAP_CRED_TO_PATH, Unum)) + data['numPaste'].append(len(uniq_num_set)) + data['simil'].append(levenRatioStr) - pprint(data) to_return = {} to_return["section"] = section to_return["action"] = action diff --git a/var/www/modules/terms/templates/credentials_tracker.html b/var/www/modules/terms/templates/credentials_tracker.html index 08fe2fec..1293aae1 100644 --- a/var/www/modules/terms/templates/credentials_tracker.html +++ b/var/www/modules/terms/templates/credentials_tracker.html @@ -1,40 +1,62 @@ - - - - - Analysis Information Leak framework Dashboard - - - - - - - - - - - - - - - - - - - - - - - {% include 'navbar.html' %} - -
-
-
-

Credential seeker

-
+ + + + + Analysis Information Leak framework Dashboard + + + + + + + + + + + + + + + + + + + + + + + + + + {% include 'navbar.html' %} + +
+
+
+

Credential seeker

+
@@ -48,36 +70,22 @@
- + - +
- - - - + + + - - {% for set in trackSet_list %} - - - - - - - - {% endfor %}
CredentialDate# line in the paste# tracked pasteUsernameSimilarity# concerned paste(s) Action
{{ set }}{{ trackSet_list_values[loop.index0][0] }}{{ trackSet_list_values[loop.index0][1] }}{{ trackSet_list_num_of_paste[loop.index0] }}

- - -

@@ -88,9 +96,10 @@
-
- -
+ +
+ +
- diff --git a/var/www/modules/terms/templates/header_terms.html b/var/www/modules/terms/templates/header_terms.html index e5970aaa..8fa38bc5 100644 --- a/var/www/modules/terms/templates/header_terms.html +++ b/var/www/modules/terms/templates/header_terms.html @@ -2,7 +2,7 @@ diff --git a/var/www/templates/header.html b/var/www/templates/header.html index 8d189318..1df983a0 100644 --- a/var/www/templates/header.html +++ b/var/www/templates/header.html @@ -20,7 +20,7 @@