CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

Draft module + web link

pull/131/head
Mokaddem 2017-07-18 16:57:15 +02:00
parent 9b8570e40b
commit 882ab88b00
5 changed files with 168 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
bin/Credential.py
View File

@ -9,6 +9,17 @@ This module is consuming the Redis-list created by the Categ module.
It apply credential regexes on paste content and warn if above a threshold.
It also split the username and store it into redis for searching purposes.
Redis organization:
uniqNumForUsername: unique number attached to unique username
uniqNumForPath: unique number attached to unique path
AllCredentials: hashed set where keys are username and value are their uniq number
AllCredentialsRev: the opposite of AllCredentials, uniqNum -> username
AllPath: hashed set where keys are path and value are their uniq number
AllPathRev: the opposite of AllPath, uniqNum -> path
splitedCred -> uniq_num (set)
"""
import time
@ -17,8 +28,19 @@ from packages import Paste
from pubsublogger import publisher
from Helper import Process
import re
import redis
from pyfaup.faup import Faup
#split username with spec. char or with upper case, distinguish start with upper
REGEX_CRED = "[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+"
REDIS_KEY_NUM_USERNAME = 'uniqNumForUsername'
REDIS_KEY_NUM_PATH = 'uniqNumForUsername'
REDIS_KEY_ALL_CRED_SET = 'AllCredentials'
REDIS_KEY_ALL_CRED_SET_REV = 'AllCredentialsRev'
REDIS_KEY_ALL_PATH_SET = 'AllPath'
REDIS_KEY_ALL_PATH_SET_REV = 'AllPathRev'
REDIS_KEY_MAP_CRED_TO_PATH = 'CredToPathMapping'
if __name__ == "__main__":
publisher.port = 6380
publisher.channel = "Script"
@ -27,6 +49,10 @@ if __name__ == "__main__":
publisher.info("Find credentials")
faup = Faup()
server_cred = redis.StrictRedis(
host=p.config.get("Redis_Level_DB_TermCred", "host"),
port=p.config.get("Redis_Level_DB_TermCred", "port"),
db=p.config.get("Redis_Level_DB_TermCred", "db"))
critical = 8
@ -37,6 +63,7 @@ if __name__ == "__main__":
message = p.get_from_set()
if message is None:
publisher.debug("Script Credential is Idling 10s")
print('sleeping 10s')
time.sleep(10)
continue
@ -44,6 +71,7 @@ if __name__ == "__main__":
if count < 5:
# Less than 5 matches from the top password list, false positive.
print("false positive:", count)
continue
paste = Paste.Paste(filepath)
@ -63,6 +91,7 @@ if __name__ == "__main__":
print('\n '.join(creds))
#num of creds above tresh, publish an alert
if len(creds) > critical:
print("========> Found more than 10 credentials in this file : {}".format(filepath))
publisher.warning(to_print)
@ -97,3 +126,31 @@ if __name__ == "__main__":
print("=======> Probably on : {}".format(', '.join(sites_set)))
else:
publisher.info(to_print)
print('found {} credentials'.format(len(creds)))
#for searching credential in cred seeker
for cred in creds:
cred = cred.split('@')[0]
#unique number attached to unique path
uniq_num_path = server_cred.incr(REDIS_KEY_ALL_PATH_SET)
print(REDIS_KEY_ALL_PATH_SET, {filepath: uniq_num_path})
server_cred.hmset(REDIS_KEY_ALL_PATH_SET, {filepath: uniq_num_path})
server_cred.hmset(REDIS_KEY_ALL_PATH_SET_REV, {uniq_num_path: filepath})
#unique number attached to unique username
uniq_num_cred = server_cred.hget(REDIS_KEY_ALL_CRED_SET, cred)
if uniq_num_cred is None: #cred do not exist, create new entries
uniq_num_cred = server_cred.incr(REDIS_KEY_NUM_USERNAME)
server_cred.hmset(REDIS_KEY_ALL_CRED_SET, {cred: uniq_num_cred})
server_cred.hmset(REDIS_KEY_ALL_CRED_SET_REV, {uniq_num_cred: cred})
server_cred.hmset(REDIS_KEY_MAP_CRED_TO_PATH, {uniq_num_cred: uniq_num_path})
splitedCred = re.findall(REGEX_CRED, cred)
print(splitedCred)
for partCred in splitedCred:
server_cred.sadd(partCred, uniq_num_cred)

5
bin/packages/config.cfg.sample
View File

@ -97,6 +97,11 @@ host = localhost
port = 6382
db = 2
[Redis_Level_DB_TermCred]
host = localhost
port = 6382
db = 5
[Redis_Level_DB]
host = localhost
port = 2016

5
var/www/modules/Flask_config.py
View File

@ -53,6 +53,11 @@ r_serv_term = redis.StrictRedis(
port=cfg.getint("Redis_Level_DB_TermFreq", "port"),
db=cfg.getint("Redis_Level_DB_TermFreq", "db"))
r_serv_cred = redis.StrictRedis(
host=cfg.get("Redis_Level_DB_TermCred", "host"),
port=cfg.getint("Redis_Level_DB_TermCred", "port"),
db=cfg.getint("Redis_Level_DB_TermCred", "db"))
r_serv_pasteName = redis.StrictRedis(
host=cfg.get("Redis_Paste_Name", "host"),
port=cfg.getint("Redis_Paste_Name", "port"),

75
var/www/modules/terms/Flask_terms.py
View File

@ -11,6 +11,7 @@ import flask
from flask import Flask, render_template, jsonify, request, Blueprint
import re
import Paste
from pprint import pprint
# ============ VARIABLES ============
import Flask_config
@ -18,6 +19,7 @@ import Flask_config
app = Flask_config.app
cfg = Flask_config.cfg
r_serv_term = Flask_config.r_serv_term
r_serv_cred = Flask_config.r_serv_cred
terms = Blueprint('terms', __name__, template_folder='templates')
@ -357,11 +359,80 @@ def credentials_management_query_paste():
cred = request.args.get('cred')
return 1
def mixUserName(supplied):
#e.g.: John Smith
terms = supplied.split()[:2]
usernames = []
if len(terms) == 1:
terms.append(' ')
#john, smith, John, Smith, JOHN, SMITH
usernames += [terms[0].lower()]
usernames += [terms[1].lower()]
usernames += [terms[0][0].upper() + terms[0][1:].lower()]
usernames += [terms[1][0].upper() + terms[1][1:].lower()]
usernames += [terms[0].upper()]
usernames += [terms[1].upper()]
#johnsmith, smithjohn, JOHNsmith, johnSMITH, SMITHjohn, smithJOHN
usernames += [(terms[0].lower() + terms[1].lower()).strip()]
usernames += [(terms[1].lower() + terms[0].lower()).strip()]
usernames += [(terms[0].upper() + terms[1].lower()).strip()]
usernames += [(terms[0].lower() + terms[1].upper()).strip()]
usernames += [(terms[1].upper() + terms[0].lower()).strip()]
usernames += [(terms[1].lower() + terms[0].upper()).strip()]
#Jsmith, JSmith, jsmith, jSmith, johnS, Js, JohnSmith, Johnsmith, johnSmith
usernames += [(terms[0][0].upper() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
usernames += [(terms[0][0].upper() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
usernames += [(terms[0][0].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
usernames += [(terms[0][0].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
usernames += [(terms[0].lower() + terms[1][0].upper()).strip()]
usernames += [(terms[0].upper() + terms[1][0].lower()).strip()]
usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
usernames += [(terms[0][0].lower() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
return usernames
@terms.route("/credentials_management_action/", methods=['GET'])
def cred_management_action():
cred = request.args.get('cred')
REGEX_CRED = '[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+'
REDIS_KEY_NUM_USERNAME = 'uniqNumForUsername'
REDIS_KEY_NUM_PATH = 'uniqNumForUsername'
REDIS_KEY_ALL_CRED_SET = 'AllCredentials'
REDIS_KEY_ALL_CRED_SET_REV = 'AllCredentialsRev'
REDIS_KEY_ALL_PATH_SET = 'AllPath'
REDIS_KEY_ALL_PATH_SET_REV = 'AllPath'
REDIS_KEY_MAP_CRED_TO_PATH = 'CredToPathMapping'
supplied = request.args.get('term')
action = request.args.get('action')
return 1
section = request.args.get('section')
#splitedCred = re.findall(REGEX_CRED, cred)
uniq_num_set = set()
if action == "seek":
possibilities = mixUserName(supplied)
for poss in possibilities:
for num in r_serv_cred.smembers(poss):
uniq_num_set.add(num)
data = {'usr': [], 'path': []}
for Unum in uniq_num_set:
data['usr'].append(r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET_REV, Unum))
data['path'].append(r_serv_cred.hget(REDIS_KEY_MAP_CRED_TO_PATH, Unum))
pprint(data)
to_return = {}
to_return["section"] = section
to_return["action"] = action
to_return["term"] = supplied
to_return["data"] = data
return jsonify(to_return)
@terms.route("/credentials_management_query/")
def cred_management_query():

54
var/www/modules/terms/templates/credentials_tracker.html
View File

@ -33,7 +33,7 @@
<div id="page-wrapper">
<div class="row">
<div class="col-lg-12">
<h1 class="page-header" data-page="page-termsfrequency" >Credentials tracking</h1>
<h1 class="page-header" data-page="page-termsfrequency" >Credential seeker</h1>
</div>
<!-- /.col-lg-12 -->
<!-- Panel OPTIONS -->
@ -43,24 +43,23 @@
<div class="col-lg-12">
<div id="panel-today" class="panel panel-success">
<div class="panel-heading">
<strong>Manage tracked credentials</strong>
<strong>Credential seeker</strong>
</div>
<div class="panel-body">
<div class="form-group input-group" style="margin-bottom: 30px;">
<span class="input-group-addon"><span class="fa fa-eye"></span></span>
<input id="followCredInput" class="form-control" placeholder="Credential to track." type="text" style="max-width: 400px;">
<button id="followTermBtn" class="btn btn-success btn-interaction" style="margin-left: 10px;" data-section="followTerm" data-action="add"> Track</button>
<input id="seekInput" class="form-control" placeholder="Credential to seek." type="text" style="max-width: 400px;">
<button id="followTermBtn" class="btn btn-success btn-interaction" style="margin-left: 10px;" data-section="seek" data-action="seek"> Seek</button>
<span id="nodata" class="alert alert-danger" style="margin-left: 10px; display: none; padding: 6px;">No data</span>
</div>
<table class="table table-striped table-bordered table-hover" id="myTable">
<thead>
<tr>
<th style="max-width: 800px;">Credential</th>
<th>Added date</th>
<th>Day occurence</th>
<th>Week occurence</th>
<th>Month occurence</th>
<th>Date</th>
<th># line in the paste</th>
<th># tracked paste</th>
<th>Action</th>
</tr>
@ -70,14 +69,12 @@
{% for set in trackSet_list %}
<tr style="background-color: #cdffca;">
<td>{{ set }}</td>
<td>{{ trackSet_list_values[loop.index0][3] }}</td>
<td>{{ trackSet_list_values[loop.index0][0] }}</td>
<td>{{ trackSet_list_values[loop.index0][1] }}</td>
<td>{{ trackSet_list_values[loop.index0][2] }}</td>
<td>{{ trackSet_list_num_of_paste[loop.index0] }}</td>
<td><p style="margin: 0px;">
<span data-toggle="modal" data-target="#mymodal" data-term="{{ set }}" ><button class="btn-link" data-toggle="tooltip" data-placement="right" title="Show concerned paste(s)"><span class="glyphicon glyphicon-info-sign"></span></button></span>
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ set }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Track this credentials" data-content="{{ set }}" data-section="followTerm" data-action="add"><span class="fa fa-eye"></span></button>
</p></td>
</tr>
{% endfor %}
@ -184,32 +181,37 @@
var curr_action = $(this).attr('data-action');
if (curr_action == "add") {
var curr_term = $('#'+curr_section+'Input').val();
} else if (curr_action == "seek") {
var curr_term = $('#'+curr_section+'Input').val();
} else {
var curr_term = $(this).attr('data-content');
}
var data_to_send = { section: curr_section, action:curr_action, term: curr_term};
if (curr_term != "") {
console.log(data_to_send);
//console.log(data_to_send);
$.get("{{ url_for('terms.cred_management_action') }}", data_to_send, function(data, status){
if(status == "success") {
var json = data;
console.log(json)
if(json.section == "followTerm") {
if(json.action == "add") {
// query data
$.get("{{ url_for('terms.cred_management_query') }}", { term: json.term, section: json.section }, function(data2, status){
var action_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"left\" title=\"Remove this term\" data-content=\"" + json.term + "\" data-section=\"followTerm\" data-action=\"delete\"><span class=\"glyphicon glyphicon-trash\"></span></button>"
table_track.row.add( [ json.term, data2[3], data2[0], data2[1], data2[2], 0, action_button ] ).draw( false );
perform_binding();
if(json.action == "add") {
} else if (json.action == "seek") {
var rep = json.data
var action_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"left\" title=\"Remove this term\" data-content=\"" + json.term + "\" data-section=\"followTerm\" data-action=\"delete\"><span class=\"glyphicon glyphicon-trash\"></span></button>"
if (rep.usr.length == 0) {
console.log('noData');
$( "#nodata" ).fadeIn( "fast", function() {
setTimeout(function() {
$( "#nodata" ).fadeOut("fast");
}, 2000);
});
} else if (json.action == "delete") {
// Find indexes of row which have the term in the first column
var index = table_track.rows().eq( 0 ).filter( function (rowIdx) {
console.log(table_track.cell( rowIdx, 0 ).data())
return table_track.cell( rowIdx, 0 ).data() === json.term;
} );
table_track.rows(index).remove().draw( false );
} else {
for(i=0; i < rep.length; i++) {
table_track.row.add( [ json.term, "xx/xx/xxxx", "xxxx", "xxxx", action_button ] ).draw( false );
}
//perform_binding();
}
}
}