CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

Added template tld. Modified URL using Faup and refactored WebStats.

pull/56/head
Mokaddem 2016-07-01 16:59:08 +02:00
parent beeeb76de9
commit 4b3101b7b6
11 changed files with 634 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
bin/LAUNCH.sh
View File

@ -107,11 +107,11 @@ function launching_scripts {
sleep 0.1
screen -S "Script" -X screen -t "Duplicate" bash -c './Duplicate.py; read x'
sleep 0.1
#screen -S "Script" -X screen -t "Attribute" bash -c './Attribute.py; read x'
screen -S "Script" -X screen -t "Attribute" bash -c './Attribute.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Line" bash -c './Line.py; read x'
sleep 0.1
#screen -S "Script" -X screen -t "DomainClassifier" bash -c './DomClassifier.py; read x'
screen -S "Script" -X screen -t "DomainClassifier" bash -c './DomClassifier.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Categ" bash -c './Categ.py; read x'
sleep 0.1
@ -139,7 +139,7 @@ function launching_scripts {
sleep 0.1
screen -S "Script" -X screen -t "Cve" bash -c './Cve.py; read x'
sleep 0.1
#screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x'
screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x'
}
#If no params, display the help

43
bin/Url.py
View File

@ -7,6 +7,8 @@ import dns.exception
from packages import Paste
from packages import lib_refine
from pubsublogger import publisher
from pyfaup.faup import Faup
import re
# Country and ASN lookup
from cymru.ip2asn.dns import DNSClient as ip2asn
@ -16,6 +18,12 @@ import ipaddress
from Helper import Process
def avoidNone(str):
if str is None:
return ""
else:
return str
if __name__ == "__main__":
publisher.port = 6380
publisher.channel = "Script"
@ -41,6 +49,7 @@ if __name__ == "__main__":
message = p.get_from_set()
prec_filename = None
faup = Faup()
url_regex = "(http|https|ftp)\://([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.(com|edu|gov|int|mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2}))(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*"
@ -53,33 +62,27 @@ if __name__ == "__main__":
PST = Paste.Paste(filename)
client = ip2asn()
for x in PST.get_regex(url_regex):
scheme, credential, subdomain, domain, host, tld, \
port, resource_path, query_string, f1, f2, f3, \
f4 = x
domains_list.append(domain)
# p.populate_set_out(x, 'Url')
temp_x = ()
for i in range(0,13):
if x[i] == '':
temp_x += ('None', )
else:
temp_x += (x[i], )
temp_scheme, temp_credential, temp_subdomain, temp_domain, temp_host, temp_tld, \
temp_port, temp_resource_path, temp_query_string, temp_f1, temp_f2, temp_f3, \
temp_f4 = temp_x
matching_url = re.search(url_regex, PST.get_p_content())
url = matching_url.group(0)
to_send = '{} {} {} {} {} {} {} {} {} {} {} {} {} {}'.format(temp_scheme, \
temp_subdomain, temp_credential, temp_domain, temp_host, temp_tld, temp_port, temp_resource_path,\
temp_query_string, temp_f1, temp_f2, temp_f3, temp_f4, PST._get_p_date())
to_send = "{} {}".format(url, PST._get_p_date())
p.populate_set_out(to_send, 'Url')
publisher.debug('{} Published'.format(x))
faup.decode(url)
domain = faup.get_domain()
subdomain = faup.get_subdomain()
f1 = None
domains_list.append(domain)
publisher.debug('{} Published'.format(url))
if f1 == "onion":
print domain
hostl = unicode(subdomain+domain)
hostl = unicode(avoidNone(subdomain)+avoidNone(domain))
try:
socket.setdefaulttimeout(2)
socket.setdefaulttimeout(1)
ip = socket.gethostbyname(unicode(hostl))
except:
# If the resolver is not giving any IPv4 address,

60
bin/WebStats.py
View File

@ -5,14 +5,24 @@
"""
import time
import datetime
import re
import redis
import os
from packages import lib_words
from pubsublogger import publisher
from packages import Paste
from Helper import Process
from pyfaup.faup import Faup
def analyse(field_name):
field = url_parsed[field_name]
if field is not None:
prev_score = r_serv1.hget(field, date)
if prev_score is not None:
r_serv1.hset(field, date, int(prev_score) + 1)
else:
r_serv1.hset(field, date, 1)
if __name__ == '__main__':
# If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
@ -37,16 +47,22 @@ if __name__ == '__main__':
db=p.config.get("Redis_Level_DB", "db"))
# FILE CURVE SECTION #
csv_path = os.path.join(os.environ['AIL_HOME'],
csv_path_proto = os.path.join(os.environ['AIL_HOME'],
p.config.get("Directories", "protocolstrending_csv"))
protocolsfile_path = os.path.join(os.environ['AIL_HOME'],
p.config.get("Directories", "protocolsfile"))
csv_path_tld = os.path.join(os.environ['AIL_HOME'],
p.config.get("Directories", "tldstrending_csv"))
tldsfile_path = os.path.join(os.environ['AIL_HOME'],
p.config.get("Directories", "tldsfile"))
faup = Faup()
generate_new_graph = False
# Endless loop getting messages from the input queue
while True:
# Get one message from the input queue
message = p.get_from_set()
generate_new_graph = False
if message is None:
if generate_new_graph:
@ -55,10 +71,15 @@ if __name__ == '__main__':
today = datetime.date.today()
year = today.year
month = today.month
lib_words.create_curve_with_word_file(r_serv1, csv_path,
lib_words.create_curve_with_word_file(r_serv1, csv_path_proto,
protocolsfile_path, year,
month)
lib_words.create_curve_with_word_file(r_serv1, csv_path_tld,
tldsfile_path, year,
month)
publisher.debug("{} queue is empty, waiting".format(config_section))
time.sleep(1)
continue
@ -66,30 +87,9 @@ if __name__ == '__main__':
else:
generate_new_graph = True
# Do something with the message from the queue
scheme, credential, subdomain, domain, host, tld, \
port, resource_path, query_string, f1, f2, f3, \
f4 , date= message.split()
prev_score = r_serv1.hget(scheme, date)
if prev_score is not None:
r_serv1.hset(scheme, date, int(prev_score) + int(score))
else:
r_serv1.hset(scheme, date, score)
url, date = message.split()
faup.decode(url)
url_parsed = faup.get()
analyse('scheme') #Scheme analysis
analyse('tld') #Tld analysis

103
files/protocolsfile
View File

@ -1,3 +1,100 @@
FTP
HTTP
HTTPS
afs
file
ftp
z39.50
z39.50r
z39.50s
vemmi
urn
nfs
dict
acap
rtspu
rtsp
rtsps
tip
pop
cid
mid
data
thismessage
service
shttp
fax
modem
tv
sip
sips
go
icap
h323
ipp
xmlrpc.beep
xmlrpc.beeps
tftp
mupdate
pres
im
mtqp
tel
iris
iris.beep
crid
snmp
tag
wais
prospero
soap.beep
soap.beeps
telnet
gopher
cap
info
dns
ldap
dav
opaquelocktoken
msrp
msrps
dtn
imap
xmpp
iax
news
nntp
snews
sms
rsync
sieve
geo
mailto
jms
mailserver
ipn
tn3270
ws
wss
xcon
xcon-userid
about
aaa
aaas
session
ni
nih
reload
ham
stun
stuns
turn
turns
http
https
coap
coaps
rtmfp
ipps
pkcs11
acct
example
vnc

263
files/tldsfile Normal file
View File

@ -0,0 +1,263 @@
com
org
net
int
edu
gov
mil
arpa
ac
ad
ae
af
ag
ai
al
am
an
ao
aq
ar
as
at
au
aw
ax
az
ba
bb
bd
be
bf
bg
bh
bi
bj
bl
bm
bn
bo
bq
br
bs
bt
bv
bw
by
bz
ca
cc
cd
cf
cg
ch
ci
ck
cl
cm
cn
co
cr
cu
cv
cw
cx
cy
cz
de
dj
dk
dm
do
dz
ec
ee
eg
eh
er
es
et
eu
fi
fj
fk
fm
fo
fr
ga
gb
gd
ge
gf
gg
gh
gi
gl
gm
gn
gp
gq
gr
gs
gt
gu
gw
gy
hk
hm
hn
hr
ht
hu
id
ie
il
im
in
io
iq
ir
is
it
je
jm
jo
jp
ke
kg
kh
ki
km
kn
kp
kr
kw
ky
kz
la
lb
lc
li
lk
lr
ls
lt
lu
lv
ly
ma
mc
md
me
mf
mg
mh
mk
ml
mm
mn
mo
mp
mq
mr
ms
mt
mu
mv
mw
mx
my
mz
na
nc
ne
nf
ng
ni
nl
no
np
nr
nu
nz
om
pa
pe
pf
pg
ph
pk
pl
pm
pn
pr
ps
pt
pw
py
qa
re
ro
rs
ru
rw
sa
sb
sc
sd
se
sg
sh
si
sj
sk
sl
sm
sn
so
sr
ss
st
su
sv
sx
sy
sz
tc
td
tf
tg
th
tj
tk
tl
tm
tn
to
tp
tr
tt
tv
tw
tz
ua
ug
uk
um
us
uy
uz
va
vc
ve
vg
vi
vn
vu
wf
ws
ye
yt
za
zm
zw

9
var/www/Flask_server.py
View File

@ -96,5 +96,14 @@ def wordstrending():
return render_template("Wordstrending.html")
@app.route("/protocolstrending/")
def protocolstrending():
return render_template("Protocolstrending.html")
@app.route("/tldstrending/")
def tldstrending():
return render_template("Tldstrending.html")
if __name__ == "__main__":
app.run(host='0.0.0.0', port=7000, threaded=True)

8
var/www/templates/Protocolstrending.html
View File

@ -24,7 +24,7 @@
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
<div class="navbar-header">
<ul class="nav navbar-nav">
<li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a><li><a href="{{ url_for('protocolsstrending') }}"><i class="glyphicon glyphicon-stats"></i> ProtocolsTrendings</a><li></ul>
<li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a></li><li><a href="{{ url_for('protocolstrending') }}"><i class="glyphicon glyphicon-stats"></i> ProtocolsTrendings</a></li><li><a href="{{ url_for('tldstrending') }}"><i class="glyphicon glyphicon-stats"></i> Top Level Domain Trending</a></li></ul>
</div>
<!-- /.navbar-top-links -->
<div class="navbar-default sidebar" role="navigation">
@ -85,7 +85,7 @@
<!-- /.panel-heading -->
<div class="panel-body">
<!-- <div id="WordTrending" style="width:100%;"></div> -->
<div id="WordTrending" style="width:100%; height:800px;"></div>
<div id="ProtocolsTrending" style="width:100%; height:800px;"></div>
</div>
<!-- /.panel-body -->
</div>
@ -95,9 +95,9 @@
<!-- /#page-wrapper -->
<script type="text/javascript">
g2 = new Dygraph(
document.getElementById("WordTrending"),
document.getElementById("ProtocolsTrending"),
// path to CSV file
"{{ url_for('static', filename='csv/wordstrendingdata.csv') }}",
"{{ url_for('static', filename='csv/protocolstrendingdata.csv') }}",
//"../csv/wordstrendingdata.csv",
//window.csv,
{

196
var/www/templates/Tldstrending.html Normal file
View File

@ -0,0 +1,196 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Analysis Information Leak framework Dashboard</title>
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
<!-- JS -->
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery-1.4.2.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
</head>
<body>
<div id="wrapper">
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
<div class="navbar-header">
<ul class="nav navbar-nav">
<li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a></li><li><a href="{{ url_for('protocolstrending') }}"><i class="glyphicon glyphicon-stats"></i> ProtocolsTrendings</a></li><li><a href="{{ url_for('tldstrending') }}"><i class="glyphicon glyphicon-stats"></i> Top Level Domain Trending</a></li></ul>
</div>
<!-- /.navbar-top-links -->
<div class="navbar-default sidebar" role="navigation">
<div class="sidebar-collapse">
<ul class="nav" id="side-menu">
<li class="sidebar-search">
<div class="input-group custom-search-form">
<input type="text" class="form-control" placeholder="Search Paste">
<span class="input-group-btn">
<button class="btn btn-default" type="button">
<i class="fa fa-search"></i>
</button>
</span>
</div>
<!-- /input-group -->
</li>
</ul>
<!-- /#side-menu -->
</div>
<!-- /.sidebar-collapse -->
<a href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
</div>
<!-- /.navbar-static-side -->
</nav>
<div id="page-wrapper">
<div class="row">
<div class="col-lg-12">
<h1 class="page-header">Top Level Domain Trending</h1>
</div>
<!-- /.col-lg-12 -->
</div>
<!-- /.row -->
<div class="row">
<div class="col-lg-12">
<div class="panel panel-default">
<div class="panel-heading">
<i class="fa fa-bar-chart-o fa-fw"></i> Top Level Domain Trending
<div class="pull-right">
<div class="btn-group">
<button type="button" class="btn btn-default btn-xs dropdown-toggle" data-toggle="dropdown">
Actions
<span class="caret"></span>
</button>
<ul class="dropdown-menu pull-right" role="menu">
<li><a href="#" id="linear">Linear Scale</a>
</li>
<li><a href="#" id="log">Log Scale</a>
</li>
<li><a href="#" id="unzoom" onclick="unzoomGraph()">Unzoom</a>
</li>
<li class="divider"></li>
<li><a href="#" id="edit_graph">Edit graph words</a>
</li>
</ul>
</div>
</div>
</div>
<!-- /.panel-heading -->
<div class="panel-body">
<!-- <div id="WordTrending" style="width:100%;"></div> -->
<div id="TldsTrending" style="width:100%; height:800px;"></div>
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.row -->
</div>
<!-- /#page-wrapper -->
<script type="text/javascript">
g2 = new Dygraph(
document.getElementById("TldsTrending"),
// path to CSV file
"{{ url_for('static', filename='csv/tldstrendingdata.csv') }}",
//"../csv/wordstrendingdata.csv",
//window.csv,
{
rollPeriod: 1,
showRoller: true,
//drawPoints: true,
//fillGraph: true,
logscale: true,
animatedZooms: true,
labelsKMB: true,
highlightCircleSize: 3,
highlightSeriesOpts: {
strokeWidth: 3,
strokeBorderWidth: 1,
highlightCircleSize: 5,
},
underlayCallback: function(canvas, area, g) {
canvas.fillStyle = "rgba(255, 193, 37, 0.5)";
function highlight_period(x_start, x_end) {
var canvas_left_x = g.toDomXCoord(x_start);
var canvas_right_x = g.toDomXCoord(x_end);
var canvas_width = canvas_right_x - canvas_left_x;
canvas.fillRect(canvas_left_x, area.y, canvas_width, area.h);
}
var min_data_x = g.getValue(0,0);
var max_data_x = g.getValue(g.numRows()-1,0);
// get day of week
var d = new Date(min_data_x);
var dow = d.getUTCDay();
var ds = d.toUTCString();
var w = min_data_x;
// starting on Sunday is a special case
if (dow == 0) {
highlight_period(w,w+12*3600*1000);
}
// find first saturday
while (dow != 5) {
w += 24*3600*1000;
d = new Date(w);
dow = d.getUTCDay();
}
// shift back 1/2 day to center highlight around the point for the day
w -= 12*3600*1000;
while (w < max_data_x) {
var start_x_highlight = w;
var end_x_highlight = w + 2*24*3600*1000;
// make sure we don't try to plot outside the graph
if (start_x_highlight < min_data_x) {
start_x_highlight = min_data_x;
}
if (end_x_highlight > max_data_x) {
end_x_highlight = max_data_x;
}
highlight_period(start_x_highlight,end_x_highlight);
// calculate start of highlight for next Saturday
w += 7*24*3600*1000;
}
}
});
onclick = function(ev) {
if (g2.isSeriesLocked()) {
g2.clearSelection();
}
else {
g2.setSelection(g2.getSelection(), g2.getHighlightSeries(), true);
}
};
g2.updateOptions({clickCallback: onclick}, true);
var linear = document.getElementById("linear");
var log = document.getElementById("log");
linear.onclick = function() { setLog(false); }
log.onclick = function() { setLog(true); }
var setLog = function(val) {
g2.updateOptions({ logscale: val });
linear.disabled = !val;
log.disabled = val;
}
function unzoomGraph() {
g2.updateOptions({
dateWindow:null,
valueRange:null
});
}
</script>
</div>
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
</body>
</html>

2
var/www/templates/Wordstrending.html
View File

@ -24,7 +24,7 @@
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
<div class="navbar-header">
<ul class="nav navbar-nav">
<li><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li class="active"><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a><li></ul>
<li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a></li><li><a href="{{ url_for('protocolstrending') }}"><i class="glyphicon glyphicon-stats"></i> ProtocolsTrendings</a></li><li><a href="{{ url_for('tldstrending') }}"><i class="glyphicon glyphicon-stats"></i> Top Level Domain Trending</a></li></ul>
</div>
<!-- /.navbar-top-links -->
<div class="navbar-default sidebar" role="navigation">

2
var/www/templates/index.html
View File

@ -35,7 +35,7 @@
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
<div class="navbar-header">
<ul class="nav navbar-nav">
<li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a><li></ul>
<li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a></li><li><a href="{{ url_for('protocolstrending') }}"><i class="glyphicon glyphicon-stats"></i> ProtocolsTrendings</a></li><li><a href="{{ url_for('tldstrending') }}"><i class="glyphicon glyphicon-stats"></i> Top Level Domain Trending</a></li></ul>
</div>
<!-- /.navbar-top-links -->
<div class="navbar-default sidebar" role="navigation">

2
var/www/templates/search.html
View File

@ -24,7 +24,7 @@
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
<div class="navbar-header">
<ul class="nav navbar-nav">
<li><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a><li></ul>
<li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a></li><li><a href="{{ url_for('protocolstrending') }}"><i class="glyphicon glyphicon-stats"></i> ProtocolsTrendings</a></li><li><a href="{{ url_for('tldstrending') }}"><i class="glyphicon glyphicon-stats"></i> Top Level Domain Trending</a></li></ul>
</div>
<!-- /.navbar-top-links -->
<div class="navbar-default sidebar" role="navigation">