diff --git a/bin/Credential.py b/bin/Credential.py index ff8f8f97..583d0457 100755 --- a/bin/Credential.py +++ b/bin/Credential.py @@ -9,6 +9,17 @@ This module is consuming the Redis-list created by the Categ module. It apply credential regexes on paste content and warn if above a threshold. +It also split the username and store it into redis for searching purposes. + +Redis organization: + uniqNumForUsername: unique number attached to unique username + uniqNumForPath: unique number attached to unique path + AllCredentials: hashed set where keys are username and value are their uniq number + AllCredentialsRev: the opposite of AllCredentials, uniqNum -> username + AllPath: hashed set where keys are path and value are their uniq number + AllPathRev: the opposite of AllPath, uniqNum -> path + splitedCred -> uniq_num (set) + """ import time @@ -17,8 +28,19 @@ from packages import Paste from pubsublogger import publisher from Helper import Process import re +import redis from pyfaup.faup import Faup +#split username with spec. char or with upper case, distinguish start with upper +REGEX_CRED = "[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+" +REDIS_KEY_NUM_USERNAME = 'uniqNumForUsername' +REDIS_KEY_NUM_PATH = 'uniqNumForUsername' +REDIS_KEY_ALL_CRED_SET = 'AllCredentials' +REDIS_KEY_ALL_CRED_SET_REV = 'AllCredentialsRev' +REDIS_KEY_ALL_PATH_SET = 'AllPath' +REDIS_KEY_ALL_PATH_SET_REV = 'AllPathRev' +REDIS_KEY_MAP_CRED_TO_PATH = 'CredToPathMapping' + if __name__ == "__main__": publisher.port = 6380 publisher.channel = "Script" @@ -27,6 +49,10 @@ if __name__ == "__main__": publisher.info("Find credentials") faup = Faup() + server_cred = redis.StrictRedis( + host=p.config.get("Redis_Level_DB_TermCred", "host"), + port=p.config.get("Redis_Level_DB_TermCred", "port"), + db=p.config.get("Redis_Level_DB_TermCred", "db")) critical = 8 @@ -37,6 +63,7 @@ if __name__ == "__main__": message = p.get_from_set() if message is None: publisher.debug("Script Credential is Idling 10s") + print('sleeping 10s') time.sleep(10) continue @@ -44,6 +71,7 @@ if __name__ == "__main__": if count < 5: # Less than 5 matches from the top password list, false positive. + print("false positive:", count) continue paste = Paste.Paste(filepath) @@ -63,6 +91,7 @@ if __name__ == "__main__": print('\n '.join(creds)) + #num of creds above tresh, publish an alert if len(creds) > critical: print("========> Found more than 10 credentials in this file : {}".format(filepath)) publisher.warning(to_print) @@ -97,3 +126,31 @@ if __name__ == "__main__": print("=======> Probably on : {}".format(', '.join(sites_set))) else: publisher.info(to_print) + print('found {} credentials'.format(len(creds))) + + + #for searching credential in cred seeker + for cred in creds: + cred = cred.split('@')[0] + + #unique number attached to unique path + uniq_num_path = server_cred.incr(REDIS_KEY_ALL_PATH_SET) + print(REDIS_KEY_ALL_PATH_SET, {filepath: uniq_num_path}) + server_cred.hmset(REDIS_KEY_ALL_PATH_SET, {filepath: uniq_num_path}) + server_cred.hmset(REDIS_KEY_ALL_PATH_SET_REV, {uniq_num_path: filepath}) + + #unique number attached to unique username + uniq_num_cred = server_cred.hget(REDIS_KEY_ALL_CRED_SET, cred) + if uniq_num_cred is None: #cred do not exist, create new entries + uniq_num_cred = server_cred.incr(REDIS_KEY_NUM_USERNAME) + server_cred.hmset(REDIS_KEY_ALL_CRED_SET, {cred: uniq_num_cred}) + server_cred.hmset(REDIS_KEY_ALL_CRED_SET_REV, {uniq_num_cred: cred}) + + server_cred.hmset(REDIS_KEY_MAP_CRED_TO_PATH, {uniq_num_cred: uniq_num_path}) + + splitedCred = re.findall(REGEX_CRED, cred) + print(splitedCred) + for partCred in splitedCred: + server_cred.sadd(partCred, uniq_num_cred) + + diff --git a/bin/packages/config.cfg.sample b/bin/packages/config.cfg.sample index 0e91a993..2e2d246d 100644 --- a/bin/packages/config.cfg.sample +++ b/bin/packages/config.cfg.sample @@ -97,6 +97,11 @@ host = localhost port = 6382 db = 2 +[Redis_Level_DB_TermCred] +host = localhost +port = 6382 +db = 5 + [Redis_Level_DB] host = localhost port = 2016 diff --git a/var/www/modules/Flask_config.py b/var/www/modules/Flask_config.py index c15e4dca..161e6130 100644 --- a/var/www/modules/Flask_config.py +++ b/var/www/modules/Flask_config.py @@ -53,6 +53,11 @@ r_serv_term = redis.StrictRedis( port=cfg.getint("Redis_Level_DB_TermFreq", "port"), db=cfg.getint("Redis_Level_DB_TermFreq", "db")) +r_serv_cred = redis.StrictRedis( + host=cfg.get("Redis_Level_DB_TermCred", "host"), + port=cfg.getint("Redis_Level_DB_TermCred", "port"), + db=cfg.getint("Redis_Level_DB_TermCred", "db")) + r_serv_pasteName = redis.StrictRedis( host=cfg.get("Redis_Paste_Name", "host"), port=cfg.getint("Redis_Paste_Name", "port"), diff --git a/var/www/modules/terms/Flask_terms.py b/var/www/modules/terms/Flask_terms.py index 75362c31..535ddafe 100644 --- a/var/www/modules/terms/Flask_terms.py +++ b/var/www/modules/terms/Flask_terms.py @@ -11,6 +11,7 @@ import flask from flask import Flask, render_template, jsonify, request, Blueprint import re import Paste +from pprint import pprint # ============ VARIABLES ============ import Flask_config @@ -18,6 +19,7 @@ import Flask_config app = Flask_config.app cfg = Flask_config.cfg r_serv_term = Flask_config.r_serv_term +r_serv_cred = Flask_config.r_serv_cred terms = Blueprint('terms', __name__, template_folder='templates') @@ -357,11 +359,80 @@ def credentials_management_query_paste(): cred = request.args.get('cred') return 1 +def mixUserName(supplied): + #e.g.: John Smith + terms = supplied.split()[:2] + usernames = [] + if len(terms) == 1: + terms.append(' ') + + #john, smith, John, Smith, JOHN, SMITH + usernames += [terms[0].lower()] + usernames += [terms[1].lower()] + usernames += [terms[0][0].upper() + terms[0][1:].lower()] + usernames += [terms[1][0].upper() + terms[1][1:].lower()] + usernames += [terms[0].upper()] + usernames += [terms[1].upper()] + + #johnsmith, smithjohn, JOHNsmith, johnSMITH, SMITHjohn, smithJOHN + usernames += [(terms[0].lower() + terms[1].lower()).strip()] + usernames += [(terms[1].lower() + terms[0].lower()).strip()] + usernames += [(terms[0].upper() + terms[1].lower()).strip()] + usernames += [(terms[0].lower() + terms[1].upper()).strip()] + usernames += [(terms[1].upper() + terms[0].lower()).strip()] + usernames += [(terms[1].lower() + terms[0].upper()).strip()] + #Jsmith, JSmith, jsmith, jSmith, johnS, Js, JohnSmith, Johnsmith, johnSmith + usernames += [(terms[0][0].upper() + terms[1][0].lower() + terms[1][1:].lower()).strip()] + usernames += [(terms[0][0].upper() + terms[1][0].upper() + terms[1][1:].lower()).strip()] + usernames += [(terms[0][0].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()] + usernames += [(terms[0][0].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()] + usernames += [(terms[0].lower() + terms[1][0].upper()).strip()] + usernames += [(terms[0].upper() + terms[1][0].lower()).strip()] + usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()] + usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()] + usernames += [(terms[0][0].lower() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()] + + return usernames + + + @terms.route("/credentials_management_action/", methods=['GET']) def cred_management_action(): - cred = request.args.get('cred') + REGEX_CRED = '[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+' + REDIS_KEY_NUM_USERNAME = 'uniqNumForUsername' + REDIS_KEY_NUM_PATH = 'uniqNumForUsername' + REDIS_KEY_ALL_CRED_SET = 'AllCredentials' + REDIS_KEY_ALL_CRED_SET_REV = 'AllCredentialsRev' + REDIS_KEY_ALL_PATH_SET = 'AllPath' + REDIS_KEY_ALL_PATH_SET_REV = 'AllPath' + REDIS_KEY_MAP_CRED_TO_PATH = 'CredToPathMapping' + + supplied = request.args.get('term') action = request.args.get('action') - return 1 + section = request.args.get('section') + + #splitedCred = re.findall(REGEX_CRED, cred) + uniq_num_set = set() + if action == "seek": + possibilities = mixUserName(supplied) + for poss in possibilities: + for num in r_serv_cred.smembers(poss): + uniq_num_set.add(num) + + data = {'usr': [], 'path': []} + for Unum in uniq_num_set: + data['usr'].append(r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET_REV, Unum)) + data['path'].append(r_serv_cred.hget(REDIS_KEY_MAP_CRED_TO_PATH, Unum)) + + pprint(data) + to_return = {} + to_return["section"] = section + to_return["action"] = action + to_return["term"] = supplied + to_return["data"] = data + + return jsonify(to_return) + @terms.route("/credentials_management_query/") def cred_management_query(): diff --git a/var/www/modules/terms/templates/credentials_tracker.html b/var/www/modules/terms/templates/credentials_tracker.html index 9f04ebea..08fe2fec 100644 --- a/var/www/modules/terms/templates/credentials_tracker.html +++ b/var/www/modules/terms/templates/credentials_tracker.html @@ -33,7 +33,7 @@
-

Credentials tracking

+

Credential seeker

@@ -43,24 +43,23 @@
- Manage tracked credentials + Credential seeker
- - + + +
- - - - + + @@ -70,14 +69,12 @@ {% for set in trackSet_list %} - - {% endfor %} @@ -184,32 +181,37 @@ var curr_action = $(this).attr('data-action'); if (curr_action == "add") { var curr_term = $('#'+curr_section+'Input').val(); + } else if (curr_action == "seek") { + var curr_term = $('#'+curr_section+'Input').val(); } else { var curr_term = $(this).attr('data-content'); } var data_to_send = { section: curr_section, action:curr_action, term: curr_term}; if (curr_term != "") { - console.log(data_to_send); + //console.log(data_to_send); $.get("{{ url_for('terms.cred_management_action') }}", data_to_send, function(data, status){ if(status == "success") { var json = data; + console.log(json) - if(json.section == "followTerm") { - if(json.action == "add") { - // query data - $.get("{{ url_for('terms.cred_management_query') }}", { term: json.term, section: json.section }, function(data2, status){ - var action_button = "" - table_track.row.add( [ json.term, data2[3], data2[0], data2[1], data2[2], 0, action_button ] ).draw( false ); - perform_binding(); + if(json.action == "add") { + + } else if (json.action == "seek") { + var rep = json.data + var action_button = "" + if (rep.usr.length == 0) { + console.log('noData'); + $( "#nodata" ).fadeIn( "fast", function() { + setTimeout(function() { + $( "#nodata" ).fadeOut("fast"); + }, 2000); }); - } else if (json.action == "delete") { - // Find indexes of row which have the term in the first column - var index = table_track.rows().eq( 0 ).filter( function (rowIdx) { - console.log(table_track.cell( rowIdx, 0 ).data()) - return table_track.cell( rowIdx, 0 ).data() === json.term; - } ); - table_track.rows(index).remove().draw( false ); + } else { + for(i=0; i < rep.length; i++) { + table_track.row.add( [ json.term, "xx/xx/xxxx", "xxxx", "xxxx", action_button ] ).draw( false ); + } + //perform_binding(); } } }
CredentialAdded dateDay occurenceWeek occurenceMonth occurenceDate# line in the paste # tracked paste Action
{{ set }}{{ trackSet_list_values[loop.index0][3] }} {{ trackSet_list_values[loop.index0][0] }} {{ trackSet_list_values[loop.index0][1] }}{{ trackSet_list_values[loop.index0][2] }} {{ trackSet_list_num_of_paste[loop.index0] }}

- +