mirror of https://github.com/CIRCL/AIL-framework
Bug fix related with redis:
Fixed typo key in redis for module creditcard and sqlinjection Modified Curve redisLvlDb server Modified Url.py so that it forwards name of protocol from saved protocolsfile Added Cache control in Flask Modified key-tab name into keys-tabpull/65/head
Mokaddem
parent
60512a30a2
commit
a80f6ca0c2
|
|
@ -68,7 +68,7 @@ if __name__ == "__main__":
|
|||
#Send to duplicate
|
||||
p.populate_set_out(filepath, 'Redis_Duplicate')
|
||||
#send to Browse_warning_paste
|
||||
p.populate_set_out('creditCard;{}'.format(filename), 'BrowseWarningPaste')
|
||||
p.populate_set_out('creditcard;{}'.format(filename), 'BrowseWarningPaste')
|
||||
else:
|
||||
publisher.info('{}CreditCard related'.format(to_print))
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -40,9 +40,9 @@ if __name__ == "__main__":
|
|||
|
||||
# REDIS #
|
||||
r_serv1 = redis.StrictRedis(
|
||||
host=p.config.get("Redis_Level_DB", "host"),
|
||||
port=p.config.get("Redis_Level_DB", "port"),
|
||||
db=p.config.get("Redis_Level_DB", "db"))
|
||||
host=p.config.get("Redis_Level_DB_Curve", "host"),
|
||||
port=p.config.get("Redis_Level_DB_Curve", "port"),
|
||||
db=p.config.get("Redis_Level_DB_Curve", "db"))
|
||||
|
||||
# FUNCTIONS #
|
||||
publisher.info("Script Curve started")
|
||||
|
|
|
|||
|
|
@ -67,6 +67,7 @@ function launching_lvldb {
|
|||
db1_y='2013'
|
||||
db2_y='2014'
|
||||
db3_y='2016'
|
||||
db4_y='3016'
|
||||
nb_db=13
|
||||
|
||||
screen -dmS "LevelDB"
|
||||
|
|
@ -78,6 +79,10 @@ function launching_lvldb {
|
|||
screen -S "LevelDB" -X screen -t "2014" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2014/ -P '$db2_y' -M '$nb_db'; read x'
|
||||
sleep 0.1
|
||||
screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db3_y' -M '$nb_db'; read x'
|
||||
|
||||
# For Curve
|
||||
sleep 0.1
|
||||
screen -S "LevelDB" -X screen -t "3016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'3016/ -P '$db4_y' -M '$nb_db'; read x'
|
||||
}
|
||||
|
||||
function launching_logs {
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@ def analyse(url, path):
|
|||
#Send to duplicate
|
||||
p.populate_set_out(path, 'Duplicate')
|
||||
#send to Browse_warning_paste
|
||||
p.populate_set_out('sqlInjectionDetection;{}'.format(path), 'BrowseWarningPaste')
|
||||
p.populate_set_out('sqlinjection;{}'.format(path), 'BrowseWarningPaste')
|
||||
else:
|
||||
print "Potential SQL injection:"
|
||||
print urllib2.unquote(url)
|
||||
|
|
|
|||
14
bin/Url.py
14
bin/Url.py
|
|
@ -3,6 +3,7 @@
|
|||
import redis
|
||||
import pprint
|
||||
import time
|
||||
import os
|
||||
import dns.exception
|
||||
from packages import Paste
|
||||
from packages import lib_refine
|
||||
|
|
@ -39,6 +40,10 @@ if __name__ == "__main__":
|
|||
port=p.config.getint("Redis_Cache", "port"),
|
||||
db=p.config.getint("Redis_Cache", "db"))
|
||||
|
||||
# Protocol file path
|
||||
protocolsfile_path = os.path.join(os.environ['AIL_HOME'],
|
||||
p.config.get("Directories", "protocolsfile"))
|
||||
|
||||
# Country to log as critical
|
||||
cc_critical = p.config.get("Url", "cc_critical")
|
||||
|
||||
|
|
@ -52,7 +57,14 @@ if __name__ == "__main__":
|
|||
prec_filename = None
|
||||
faup = Faup()
|
||||
|
||||
url_regex = "(http|https|ftp)\://([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.(com|edu|gov|int|mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2}))(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*"
|
||||
# Get all uri from protocolsfile (Used for Curve)
|
||||
uri_scheme = ""
|
||||
with open(protocolsfile_path, 'r') as scheme_file:
|
||||
for scheme in scheme_file:
|
||||
uri_scheme += scheme[:-1]+"|"
|
||||
uri_scheme = uri_scheme[:-1]
|
||||
|
||||
url_regex = "("+uri_scheme+")\://([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.(com|edu|gov|int|mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2}))(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*"
|
||||
|
||||
while True:
|
||||
if message is not None:
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ def analyse(server, field_name, date, url_parsed):
|
|||
prev_score = server.hget(field, date)
|
||||
if prev_score is not None:
|
||||
server.hset(field, date, int(prev_score) + 1)
|
||||
|
||||
else:
|
||||
server.hset(field, date, 1)
|
||||
if field_name == "domain": #save domain in a set for the monthly plot
|
||||
|
|
@ -67,7 +68,6 @@ def compute_progression(server, field_name, num_day, url_parsed):
|
|||
divisor = value_list_reversed[i-1] if value_list_reversed[i-1] != 0 else 1
|
||||
keyword_increase += value_list_reversed[i] / divisor
|
||||
|
||||
|
||||
# filter
|
||||
if (keyword_total_sum > threshold_total_sum) and (keyword_increase > threshold_increase):
|
||||
|
||||
|
|
|
|||
|
|
@ -164,7 +164,6 @@ def showpaste(content_range):
|
|||
def getPastebyType(server, module_name):
|
||||
all_path = []
|
||||
for path in server.smembers('WARNING_'+module_name):
|
||||
#all_path.append("/home/mokaddem/AIL-framework/PASTES/archive/paste.debian.net/2016/06/30/771058.gz")
|
||||
all_path.append(path)
|
||||
return all_path
|
||||
|
||||
|
|
@ -198,7 +197,16 @@ def get_top_relevant_data(server, module_name):
|
|||
member_set.insert(0, ("passed_days", days))
|
||||
return member_set
|
||||
|
||||
|
||||
# ========= CACHE CONTROL ========
|
||||
@app.after_request
|
||||
def add_header(response):
|
||||
"""
|
||||
Add headers to both force latest IE rendering engine or Chrome Frame,
|
||||
and also to cache the rendered page for 10 minutes.
|
||||
"""
|
||||
response.headers['X-UA-Compatible'] = 'IE=Edge,chrome=1'
|
||||
response.headers['Cache-Control'] = 'public, max-age=0'
|
||||
return response
|
||||
|
||||
# ============ ROUTES ============
|
||||
|
||||
|
|
|
|||
|
|
@ -112,19 +112,21 @@ function Graph(id_pannel, path, header_size){
|
|||
|
||||
// display the top headers
|
||||
function setVis(max_display){
|
||||
headings = this.graph.getLabels();
|
||||
var headings = this.graph.getLabels();
|
||||
headings.splice(0,1);
|
||||
var sorted_list = new Array();
|
||||
today = new Date().getDate()-1; // Take the top from yesterday so that we can see the current evolution
|
||||
var today = new Date().getDate();
|
||||
//today = new Date().getDate()-1; // Could take the top from yesterday so that we can see the current evolution
|
||||
for( i=0; i<headings.length; i++){
|
||||
the_heading = headings[i];
|
||||
//console.log('heading='+the_heading+' tab['+(today-1)+']['+(parseInt(i)+1)+']='+g.getValue(today-1, parseInt(i)+1));
|
||||
var the_heading = headings[i];
|
||||
//console.log('heading='+the_heading+' tab['+(today)+']['+(parseInt(i)+1)+']='+this.graph.getValue(today-1, parseInt(i)+1));
|
||||
sorted_list.push({dom: the_heading, val: this.graph.getValue(today-1, parseInt(i)+1), index: parseInt(i)});
|
||||
}
|
||||
sorted_list.sort(function(a,b) {
|
||||
return b.val - a.val;
|
||||
});
|
||||
|
||||
|
||||
var display_list = sorted_list.slice(0, max_display);
|
||||
for( i=0; i<display_list.length; i++){
|
||||
this.graph.setVisibility(display_list[i].index, true);
|
||||
|
|
|
|||
|
|
@ -5,6 +5,10 @@
|
|||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
|
||||
<meta http-equiv="Cache-Control" content="no-cache, no-store, must-revalidate" />
|
||||
<meta http-equiv="Pragma" content="no-cache" />
|
||||
<meta http-equiv="Expires" content="0" />
|
||||
|
||||
<title>Analysis Information Leak framework Dashboard</title>
|
||||
|
||||
<!-- Core CSS -->
|
||||
|
|
|
|||
|
|
@ -99,7 +99,7 @@
|
|||
<li name='nav-pan'><a data-toggle="tab" href="#creditcard-tab" data-attribute-name="creditcard" data-panel="creditcard-panel">Credit cards</a></li>
|
||||
<li name='nav-pan'><a data-toggle="tab" href="#sqlinjection-tab" data-attribute-name="sqlinjection" data-panel="sqlinjection-panel">SQL injections</a></li>
|
||||
<li name='nav-pan'><a data-toggle="tab" href="#cve-tab" data-attribute-name="cve" data-panel="cve-panel">CVEs</a></li>
|
||||
<li name='nav-pan'><a data-toggle="tab" href="#key-tab" data-attribute-name="key" data-panel="key-panel">Keys</a></li>
|
||||
<li name='nav-pan'><a data-toggle="tab" href="#keys-tab" data-attribute-name="keys" data-panel="keys-panel">Keys</a></li>
|
||||
<li name='nav-pan'><a data-toggle="tab" href="#mail-tab" data-attribute-name="mail" data-panel="mail-panel">Mails</a></li>
|
||||
<li name='nav-pan'><a data-toggle="tab" href="#phone-tab" data-attribute-name="phone" data-panel="phone-panel">Phones</a></li>
|
||||
</ul>
|
||||
|
|
@ -119,7 +119,7 @@
|
|||
<div class="col-lg-12 tab-pane fade" id="cve-tab">
|
||||
<img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" style="margin: 4px;">
|
||||
</div>
|
||||
<div class="col-lg-12 tab-pane fade" id="key-tab">
|
||||
<div class="col-lg-12 tab-pane fade" id="keys-tab">
|
||||
<img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" style="margin: 4px;">
|
||||
</div>
|
||||
<div class="col-lg-12 tab-pane fade" id="mail-tab">
|
||||
|
|
@ -150,7 +150,6 @@ var previous_tab = $('[data-attribute-name="credential');
|
|||
var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_for('static', filename='image/loading.gif') }}\" height='26' width='26' style='margin: 4px;'>";
|
||||
|
||||
$('.nav-tabs a').on('shown.bs.tab', function(event){
|
||||
console.log(event.target);
|
||||
var dataPath = $(event.target).attr('data-attribute-name');
|
||||
$.get("{{ url_for('importantPasteByModule') }}"+"?moduleName="+dataPath, function(data, status){
|
||||
var currentTab = $('[name].active').children();
|
||||
|
|
|
|||
Loading…
Reference in New Issue