#!/usr/bin/env python3.5 # -*-coding:UTF-8 -* ''' Flask functions and routes for the trending modules page note: The matching of credential against supplied credential is done using Levenshtein distance ''' import redis import datetime import calendar import flask from flask import Flask, render_template, jsonify, request, Blueprint import re import Paste from pprint import pprint import Levenshtein # ============ VARIABLES ============ import Flask_config app = Flask_config.app cfg = Flask_config.cfg r_serv_term = Flask_config.r_serv_term r_serv_cred = Flask_config.r_serv_cred terms = Blueprint('terms', __name__, template_folder='templates') '''TERM''' DEFAULT_MATCH_PERCENT = 50 #tracked TrackedTermsSet_Name = "TrackedSetTermSet" TrackedTermsDate_Name = "TrackedTermDate" #black BlackListTermsDate_Name = "BlackListTermDate" BlackListTermsSet_Name = "BlackListSetTermSet" #regex TrackedRegexSet_Name = "TrackedRegexSet" TrackedRegexDate_Name = "TrackedRegexDate" #set TrackedSetSet_Name = "TrackedSetSet" TrackedSetDate_Name = "TrackedSetDate" # notifications enabled/disabled # same value as in `bin/NotificationHelper.py` TrackedTermsNotificationEnabled_Name = "TrackedNotifications" # associated notification email addresses for a specific term` # same value as in `bin/NotificationHelper.py` # Keys will be e.g. TrackedNotificationEmails_ TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_" '''CRED''' REGEX_CRED = '[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+' REDIS_KEY_NUM_USERNAME = 'uniqNumForUsername' REDIS_KEY_NUM_PATH = 'uniqNumForUsername' REDIS_KEY_ALL_CRED_SET = 'AllCredentials' REDIS_KEY_ALL_CRED_SET_REV = 'AllCredentialsRev' REDIS_KEY_ALL_PATH_SET = 'AllPath' REDIS_KEY_ALL_PATH_SET_REV = 'AllPathRev' REDIS_KEY_MAP_CRED_TO_PATH = 'CredToPathMapping' # ============ FUNCTIONS ============ def Term_getValueOverRange(word, startDate, num_day, per_paste=""): passed_days = 0 oneDay = 60*60*24 to_return = [] curr_to_return = 0 for timestamp in range(startDate, startDate - max(num_day)*oneDay, -oneDay): value = r_serv_term.hget(per_paste+str(timestamp), word) curr_to_return += int(value.decode('utf8')) if value is not None else 0 for i in num_day: if passed_days == i-1: to_return.append(curr_to_return) passed_days += 1 return to_return #Mix suplied username, if extensive is set, slice username(s) with different windows def mixUserName(supplied, extensive=False): #e.g.: John Smith terms = supplied.split()[:2] usernames = [] if len(terms) == 1: terms.append(' ') #john, smith, John, Smith, JOHN, SMITH usernames += [terms[0].lower()] usernames += [terms[1].lower()] usernames += [terms[0][0].upper() + terms[0][1:].lower()] usernames += [terms[1][0].upper() + terms[1][1:].lower()] usernames += [terms[0].upper()] usernames += [terms[1].upper()] #johnsmith, smithjohn, JOHNsmith, johnSMITH, SMITHjohn, smithJOHN usernames += [(terms[0].lower() + terms[1].lower()).strip()] usernames += [(terms[1].lower() + terms[0].lower()).strip()] usernames += [(terms[0].upper() + terms[1].lower()).strip()] usernames += [(terms[0].lower() + terms[1].upper()).strip()] usernames += [(terms[1].upper() + terms[0].lower()).strip()] usernames += [(terms[1].lower() + terms[0].upper()).strip()] #Jsmith, JSmith, jsmith, jSmith, johnS, Js, JohnSmith, Johnsmith, johnSmith usernames += [(terms[0][0].upper() + terms[1][0].lower() + terms[1][1:].lower()).strip()] usernames += [(terms[0][0].upper() + terms[1][0].upper() + terms[1][1:].lower()).strip()] usernames += [(terms[0][0].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()] usernames += [(terms[0][0].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()] usernames += [(terms[0].lower() + terms[1][0].upper()).strip()] usernames += [(terms[0].upper() + terms[1][0].lower()).strip()] usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()] usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()] usernames += [(terms[0][0].lower() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()] if not extensive: return usernames #Slice the supplied username(s) mixedSupplied = supplied.replace(' ','') minWindow = 3 if len(mixedSupplied)/2 < 4 else len(mixedSupplied)/2 for winSize in range(3,len(mixedSupplied)): for startIndex in range(0, len(mixedSupplied)-winSize): usernames += [mixedSupplied[startIndex:startIndex+winSize]] filtered_usernames = [] for usr in usernames: if len(usr) > 2: filtered_usernames.append(usr) return filtered_usernames # ============ ROUTES ============ @terms.route("/terms_management/") def terms_management(): per_paste = request.args.get('per_paste') if per_paste == "1" or per_paste is None: per_paste_text = "per_paste_" per_paste = 1 else: per_paste_text = "" per_paste = 0 today = datetime.datetime.now() today = today.replace(hour=0, minute=0, second=0, microsecond=0) today_timestamp = calendar.timegm(today.timetuple()) # Map tracking if notifications are enabled for a specific term notificationEnabledDict = {} # Maps a specific term to the associated email addresses notificationEMailTermMapping = {} #Regex trackReg_list = [] trackReg_list_values = [] trackReg_list_num_of_paste = [] for tracked_regex in r_serv_term.smembers(TrackedRegexSet_Name): notificationEMailTermMapping[tracked_regex] = "\n".join(r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_regex)) if tracked_regex not in notificationEnabledDict: notificationEnabledDict[tracked_regex] = False trackReg_list.append(tracked_regex) value_range = Term_getValueOverRange(tracked_regex, today_timestamp, [1, 7, 31], per_paste=per_paste_text) term_date = r_serv_term.hget(TrackedRegexDate_Name, tracked_regex) set_paste_name = "regex_" + tracked_regex trackReg_list_num_of_paste.append(r_serv_term.scard(set_paste_name)) term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded" value_range.append(term_date) trackReg_list_values.append(value_range) if tracked_regex in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name): notificationEnabledDict[tracked_regex] = True #Set trackSet_list = [] trackSet_list_values = [] trackSet_list_num_of_paste = [] for tracked_set in r_serv_term.smembers(TrackedSetSet_Name): notificationEMailTermMapping[tracked_set] = "\n".join(r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_set)) if tracked_set not in notificationEnabledDict: notificationEnabledDict[tracked_set] = False trackSet_list.append(tracked_set) value_range = Term_getValueOverRange(tracked_set, today_timestamp, [1, 7, 31], per_paste=per_paste_text) term_date = r_serv_term.hget(TrackedSetDate_Name, tracked_set) set_paste_name = "set_" + tracked_set trackSet_list_num_of_paste.append(r_serv_term.scard(set_paste_name)) term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded" value_range.append(term_date) trackSet_list_values.append(value_range) if tracked_set in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name): notificationEnabledDict[tracked_set] = True #Tracked terms track_list = [] track_list_values = [] track_list_num_of_paste = [] for tracked_term in r_serv_term.smembers(TrackedTermsSet_Name): notificationEMailTermMapping[tracked_term] = "\n".join(r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_term)) if tracked_term not in notificationEnabledDict: notificationEnabledDict[tracked_term] = False track_list.append(tracked_term) value_range = Term_getValueOverRange(tracked_term, today_timestamp, [1, 7, 31], per_paste=per_paste_text) term_date = r_serv_term.hget(TrackedTermsDate_Name, tracked_term) set_paste_name = "tracked_" + tracked_term track_list_num_of_paste.append(r_serv_term.scard(set_paste_name)) term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded" value_range.append(term_date) track_list_values.append(value_range) if tracked_term in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name): notificationEnabledDict[tracked_term] = True #blacklist terms black_list = [] for blacked_term in r_serv_term.smembers(BlackListTermsSet_Name): term_date = r_serv_term.hget(BlackListTermsDate_Name, blacked_term) term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded" black_list.append([blacked_term, term_date]) return render_template("terms_management.html", black_list=black_list, track_list=track_list, trackReg_list=trackReg_list, trackSet_list=trackSet_list, track_list_values=track_list_values, track_list_num_of_paste=track_list_num_of_paste, trackReg_list_values=trackReg_list_values, trackReg_list_num_of_paste=trackReg_list_num_of_paste, trackSet_list_values=trackSet_list_values, trackSet_list_num_of_paste=trackSet_list_num_of_paste, per_paste=per_paste, notificationEnabledDict=notificationEnabledDict, notificationEMailTermMapping=notificationEMailTermMapping) @terms.route("/terms_management_query_paste/") def terms_management_query_paste(): term = request.args.get('term') paste_info = [] # check if regex or not if term.startswith('/') and term.endswith('/'): set_paste_name = "regex_" + term track_list_path = (r_serv_term.smembers(set_paste_name)).decode('utf8') elif term.startswith('\\') and term.endswith('\\'): set_paste_name = "set_" + term track_list_path = (r_serv_term.smembers(set_paste_name)).decode('utf8') else: set_paste_name = "tracked_" + term track_list_path = (r_serv_term.smembers(set_paste_name)).decode('utf8') for path in track_list_path: paste = Paste.Paste(path) p_date = str(paste._get_p_date()) p_date = p_date[6:]+'/'+p_date[4:6]+'/'+p_date[0:4] p_source = paste.p_source p_encoding = paste._get_p_encoding() p_size = paste.p_size p_mime = paste.p_mime p_lineinfo = paste.get_lines_info() p_content = paste.get_p_content() if p_content != 0: p_content = p_content[0:400] paste_info.append({"path": path, "date": p_date, "source": p_source, "encoding": p_encoding, "size": p_size, "mime": p_mime, "lineinfo": p_lineinfo, "content": p_content}) return jsonify(paste_info) @terms.route("/terms_management_query/") def terms_management_query(): TrackedTermsDate_Name = "TrackedTermDate" BlackListTermsDate_Name = "BlackListTermDate" term = request.args.get('term') section = request.args.get('section') today = datetime.datetime.now() today = today.replace(hour=0, minute=0, second=0, microsecond=0) today_timestamp = calendar.timegm(today.timetuple()) value_range = Term_getValueOverRange(term, today_timestamp, [1, 7, 31]) if section == "followTerm": term_date = r_serv_term.hget(TrackedTermsDate_Name, term) elif section == "blacklistTerm": term_date = r_serv_term.hget(BlackListTermsDate_Name, term) term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded" value_range.append(str(term_date)) return jsonify(value_range) @terms.route("/terms_management_action/", methods=['GET']) def terms_management_action(): today = datetime.datetime.now() today = today.replace(microsecond=0) today_timestamp = calendar.timegm(today.timetuple()) section = request.args.get('section') action = request.args.get('action') term = request.args.get('term') notificationEmailsParam = request.args.get('emailAddresses') if action is None or term is None or notificationEmailsParam is None: return "None" else: if section == "followTerm": if action == "add": # Strip all whitespace notificationEmailsParam = "".join(notificationEmailsParam.split()) # Make a list of all passed email addresses notificationEmails = notificationEmailsParam.split(",") validNotificationEmails = [] # check for valid email addresses for email in notificationEmails: # Really basic validation: # has exactly one @ sign, and at least one . in the part after the @ if re.match(r"[^@]+@[^@]+\.[^@]+", email): validNotificationEmails.append(email) # check if regex/set or simple term #regex if term.startswith('/') and term.endswith('/'): r_serv_term.sadd(TrackedRegexSet_Name, term) r_serv_term.hset(TrackedRegexDate_Name, term, today_timestamp) # add all valid emails to the set for email in validNotificationEmails: r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term, email) # enable notifications by default r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term) #set elif term.startswith('\\') and term.endswith('\\'): tab_term = term[1:-1] perc_finder = re.compile("\[[0-9]{1,3}\]").search(tab_term) if perc_finder is not None: match_percent = perc_finder.group(0)[1:-1] set_to_add = term else: match_percent = DEFAULT_MATCH_PERCENT set_to_add = "\\" + tab_term[:-1] + ", [{}]]\\".format(match_percent) r_serv_term.sadd(TrackedSetSet_Name, set_to_add) r_serv_term.hset(TrackedSetDate_Name, set_to_add, today_timestamp) # add all valid emails to the set for email in validNotificationEmails: r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + set_to_add, email) # enable notifications by default r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, set_to_add) #simple term else: r_serv_term.sadd(TrackedTermsSet_Name, term.lower()) r_serv_term.hset(TrackedTermsDate_Name, term.lower(), today_timestamp) # add all valid emails to the set for email in validNotificationEmails: r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term.lower(), email) # enable notifications by default r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term.lower()) elif action == "toggleEMailNotification": # get the current state if term in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name): # remove it r_serv_term.srem(TrackedTermsNotificationEnabled_Name, term.lower()) else: # add it r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term.lower()) #del action else: if term.startswith('/') and term.endswith('/'): r_serv_term.srem(TrackedRegexSet_Name, term) r_serv_term.hdel(TrackedRegexDate_Name, term) elif term.startswith('\\') and term.endswith('\\'): r_serv_term.srem(TrackedSetSet_Name, term) #print(term) r_serv_term.hdel(TrackedSetDate_Name, term) else: r_serv_term.srem(TrackedTermsSet_Name, term.lower()) r_serv_term.hdel(TrackedTermsDate_Name, term.lower()) # delete the associated notification emails too r_serv_term.delete(TrackedTermsNotificationEmailsPrefix_Name + term) elif section == "blacklistTerm": if action == "add": r_serv_term.sadd(BlackListTermsSet_Name, term.lower()) r_serv_term.hset(BlackListTermsDate_Name, term, today_timestamp) else: r_serv_term.srem(BlackListTermsSet_Name, term.lower()) else: return "None" to_return = {} to_return["section"] = section to_return["action"] = action to_return["term"] = term return jsonify(to_return) @terms.route("/terms_plot_tool/") def terms_plot_tool(): term = request.args.get('term') if term is not None: return render_template("terms_plot_tool.html", term=term) else: return render_template("terms_plot_tool.html", term="") @terms.route("/terms_plot_tool_data/") def terms_plot_tool_data(): oneDay = 60*60*24 range_start = datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_start')))) if request.args.get('range_start') is not None else 0; range_start = range_start.replace(hour=0, minute=0, second=0, microsecond=0) range_start = calendar.timegm(range_start.timetuple()) range_end = datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_end')))) if request.args.get('range_end') is not None else 0; range_end = range_end.replace(hour=0, minute=0, second=0, microsecond=0) range_end = calendar.timegm(range_end.timetuple()) term = request.args.get('term') per_paste = request.args.get('per_paste') if per_paste == "1" or per_paste is None: per_paste = "per_paste_" else: per_paste = "" if term is None: return "None" else: value_range = [] for timestamp in range(range_start, range_end+oneDay, oneDay): value = r_serv_term.hget(per_paste+str(timestamp), term) curr_value_range = int(value) if value is not None else 0 value_range.append([timestamp, curr_value_range]) value_range.insert(0,term) return jsonify(value_range) @terms.route("/terms_plot_top/") def terms_plot_top(): per_paste = request.args.get('per_paste') per_paste = per_paste if per_paste is not None else 1 return render_template("terms_plot_top.html", per_paste=per_paste) @terms.route("/terms_plot_top_data/") def terms_plot_top_data(): oneDay = 60*60*24 today = datetime.datetime.now() today = today.replace(hour=0, minute=0, second=0, microsecond=0) today_timestamp = calendar.timegm(today.timetuple()) per_paste = request.args.get('per_paste') if per_paste == "1" or per_paste is None: per_paste = "per_paste_" else: per_paste = "" set_day = per_paste + "TopTermFreq_set_day_" + str(today_timestamp) set_week = per_paste + "TopTermFreq_set_week"; set_month = per_paste + "TopTermFreq_set_month"; the_set = per_paste + request.args.get('set') num_day = int(request.args.get('num_day')) if the_set is None: return "None" else: to_return = [] if "TopTermFreq_set_day" in the_set: the_set += "_" + str(today_timestamp) for term, tot_value in r_serv_term.zrevrangebyscore(the_set, '+inf', '-inf', withscores=True, start=0, num=20): position = {} position['day'] = r_serv_term.zrevrank(set_day, term) position['day'] = position['day']+1 if position['day'] is not None else "<20" position['week'] = r_serv_term.zrevrank(set_week, term) position['week'] = position['week']+1 if position['week'] is not None else "<20" position['month'] = r_serv_term.zrevrank(set_month, term) position['month'] = position['month']+1 if position['month'] is not None else "<20" value_range = [] for timestamp in range(today_timestamp, today_timestamp - num_day*oneDay, -oneDay): value = r_serv_term.hget(per_paste+str(timestamp), term) curr_value_range = int(value) if value is not None else 0 value_range.append([timestamp, curr_value_range]) to_return.append([term.decode('utf8'), value_range, tot_value, position]) return jsonify(to_return) @terms.route("/credentials_tracker/") def credentials_tracker(): return render_template("credentials_tracker.html") @terms.route("/credentials_management_query_paste/", methods=['GET', 'POST']) def credentials_management_query_paste(): cred = request.args.get('cred') allPath = request.json['allPath'] paste_info = [] for pathNum in allPath: path = r_serv_cred.hget(REDIS_KEY_ALL_PATH_SET_REV, pathNum) paste = Paste.Paste(path) p_date = str(paste._get_p_date()) p_date = p_date[6:]+'/'+p_date[4:6]+'/'+p_date[0:4] p_source = paste.p_source p_encoding = paste._get_p_encoding() p_size = paste.p_size p_mime = paste.p_mime p_lineinfo = paste.get_lines_info() p_content = paste.get_p_content().decode('utf-8', 'ignore') if p_content != 0: p_content = p_content[0:400] paste_info.append({"path": path, "date": p_date, "source": p_source, "encoding": p_encoding, "size": p_size, "mime": p_mime, "lineinfo": p_lineinfo, "content": p_content}) return jsonify(paste_info) @terms.route("/credentials_management_action/", methods=['GET']) def cred_management_action(): supplied = request.args.get('term') action = request.args.get('action') section = request.args.get('section') extensive = request.args.get('extensive') extensive = True if extensive == "true" else False if extensive: #collectDico AllUsernameInRedis = r_serv_cred.hgetall(REDIS_KEY_ALL_CRED_SET).keys() uniq_num_set = set() if action == "seek": possibilities = mixUserName(supplied, extensive) for poss in possibilities: num = r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET, poss) if num is not None: uniq_num_set.add(num) for num in r_serv_cred.smembers(poss): uniq_num_set.add(num) #Extensive /!\ if extensive: iter_num = 0 tot_iter = len(AllUsernameInRedis)*len(possibilities) for tempUsername in AllUsernameInRedis: tempUsername = tempUsername.decode('utf8') for poss in possibilities: #FIXME print progress if(iter_num % int(tot_iter/20) == 0): #print("searching: {}% done".format(int(iter_num/tot_iter*100)), sep=' ', end='\r', flush=True) print("searching: {}% done".format(float(iter_num)/float(tot_iter)*100)) iter_num += 1 if poss in tempUsername: num = (r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET, tempUsername)).decode('utf8') if num is not None: uniq_num_set.add(num) for num in r_serv_cred.smembers(tempUsername): uniq_num_set.add(num) data = {'usr': [], 'path': [], 'numPaste': [], 'simil': []} for Unum in uniq_num_set: levenRatio = 2.0 username = (r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET_REV, Unum)).decode('utf8') # Calculate Levenshtein distance, ignore negative ratio supp_splitted = supplied.split() supp_mixed = supplied.replace(' ','') supp_splitted.append(supp_mixed) for indiv_supplied in supp_splitted: levenRatio = float(Levenshtein.ratio(indiv_supplied, username)) levenRatioStr = "{:.1%}".format(levenRatio) data['usr'].append(username) try: Unum = Unum.decode('utf8') except: pass allPathNum = list(r_serv_cred.smembers(REDIS_KEY_MAP_CRED_TO_PATH+'_'+Unum)) # decode bytes allPathNum_str = [] for p in allPathNum: allPathNum_str.append(p.decode('utf8')) data['path'].append(allPathNum_str) data['numPaste'].append(len(allPathNum_str)) data['simil'].append(levenRatioStr) to_return = {} to_return["section"] = section to_return["action"] = action to_return["term"] = supplied to_return["data"] = data return jsonify(to_return) # ========= REGISTRATION ========= app.register_blueprint(terms)