CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

Getting Started: Initial configuration working on the laptop

pull/56/head
Mokaddem 2016-06-30 14:36:47 +02:00
parent a44bdfa7d4
commit 3dc356dc5e
7 changed files with 43 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bin/Categ.py
View File

@ -66,7 +66,7 @@ if __name__ == "__main__":
# FUNCTIONS #
publisher.info("Script Categ started")
categories = ['CreditCards', 'Mail', 'Onion', 'Web', 'Credential']
categories = ['CreditCards', 'Mail', 'Onion', 'Web', 'Credential', 'Cve']
tmp_dict = {}
for filename in categories:
bname = os.path.basename(filename)

2
bin/DomClassifier.py
View File

@ -39,7 +39,7 @@ def main():
if message is not None:
PST = Paste.Paste(message)
else:
publisher.debug("Script DomClassifier is idling 10s")
publisher.debug("Script DomClassifier is idling 1s")
time.sleep(1)
continue
paste = PST.get_p_content()

5
bin/Duplicate.py
View File

@ -33,11 +33,12 @@ if __name__ == "__main__":
# DB OBJECT & HASHS ( DISK )
# FIXME increase flexibility
dico_redis = {}
for year in xrange(2013, 2015):
for year in xrange(2013, 2017):
for month in xrange(0, 16):
dico_redis[str(year)+str(month).zfill(2)] = redis.StrictRedis(
host=p.config.get("Redis_Level_DB", "host"), port=year,
db=month)
#print("dup: "+str(year)+str(month).zfill(2)+"\n")
# FUNCTIONS #
publisher.info("Script duplicate started")
@ -81,7 +82,7 @@ if __name__ == "__main__":
bloop_path_set.add(filebloompath)
# UNIQUE INDEX HASHS TABLE
r_serv0 = dico_redis["201300"]
r_serv0 = dico_redis["201600"]
r_serv0.incr("current_index")
index = r_serv0.get("current_index")+str(PST.p_date)
# HASHTABLES PER MONTH (because of r_serv1 changing db)

17
bin/LAUNCH.sh
View File

@ -66,6 +66,7 @@ function launching_lvldb {
lvdbdir="${AIL_HOME}/LEVEL_DB_DATA/"
db1_y='2013'
db2_y='2014'
db3_y='2016'
nb_db=13
screen -dmS "LevelDB"
@ -75,6 +76,8 @@ function launching_lvldb {
screen -S "LevelDB" -X screen -t "2013" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2013/ -P '$db1_y' -M '$nb_db'; read x'
sleep 0.1
screen -S "LevelDB" -X screen -t "2014" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2014/ -P '$db2_y' -M '$nb_db'; read x'
sleep 0.1
screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db3_y' -M '$nb_db'; read x'
}
function launching_logs {
@ -104,11 +107,11 @@ function launching_scripts {
sleep 0.1
screen -S "Script" -X screen -t "Duplicate" bash -c './Duplicate.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Attribute" bash -c './Attribute.py; read x'
#screen -S "Script" -X screen -t "Attribute" bash -c './Attribute.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Line" bash -c './Line.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "DomainClassifier" bash -c './DomClassifier.py; read x'
#screen -S "Script" -X screen -t "DomainClassifier" bash -c './DomClassifier.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Categ" bash -c './Categ.py; read x'
sleep 0.1
@ -127,6 +130,16 @@ function launching_scripts {
screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Keys" bash -c './Keys.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Phone" bash -c './Phone.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Release" bash -c './Release.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Cve" bash -c './Cve.py; read x'
sleep 0.1
#screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x'
}
#If no params, display the help

16
bin/Url.py
View File

@ -57,7 +57,21 @@ if __name__ == "__main__":
port, resource_path, query_string, f1, f2, f3, \
f4 = x
domains_list.append(domain)
p.populate_set_out(x, 'Url')
# p.populate_set_out(x, 'Url')
temp_x = ()
for i in range(0,13):
if x[i] == '':
temp_x += ('None', )
else:
temp_x += (x[i], )
temp_scheme, temp_credential, temp_subdomain, temp_domain, temp_host, temp_tld, \
temp_port, temp_resource_path, temp_query_string, temp_f1, temp_f2, temp_f3, \
temp_f4 = temp_x
to_send = '{} {} {} {} {} {} {} {} {} {} {} {} {} {}'.format(temp_scheme, \
temp_subdomain, temp_credential, temp_domain, temp_host, temp_tld, temp_port, temp_resource_path,\
temp_query_string, temp_f1, temp_f2, temp_f3, temp_f4, PST._get_p_date())
p.populate_set_out(to_send , 'Url')
publisher.debug('{} Published'.format(x))
if f1 == "onion":

8
bin/packages/modules.cfg
View File

@ -27,7 +27,7 @@ subscribe = Redis_Words
[Categ]
subscribe = Redis_Global
publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Redis_SourceCode
publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Redis_SourceCode,Redis_Cve
[CreditCards]
subscribe = Redis_CreditCards
@ -47,12 +47,18 @@ subscribe = Redis_ValidOnion
subscribe = Redis_Web
publish = Redis_Url,ZMQ_Url
[WebStats]
subscribe = Redis_Url
[Release]
subscribe = Redis_Global
[Credential]
subscribe = Redis_Credential
[Cve]
subscribe = Redis_Cve
[Phone]
subscribe = Redis_Global

2
var/www/update_thirdparty.sh
View File

@ -17,7 +17,7 @@ mv temp/startbootstrap-sb-admin-2-1.0.2 temp/sb-admin-2
JQVERSION="1.11.1"
wget http://code.jquery.com/jquery-${JQVERSION}.js -O ./static/js/jquery.js
wget https://collabdev.googlecode.com/svn-history/r5/trunk/static/js/jquery.timers-1.0.0.js -O ./static/js/jquery.timers-1.0.0.js
#wget https://collabdev.googlecode.com/svn-history/r5/trunk/static/js/jquery.timers-1.0.0.js -O ./static/js/jquery.timers-1.0.0.js
#Here to fix an error about an hard dependency in a obscur script of bootstrap..
wget http://code.jquery.com/jquery-1.4.2.js -O ./static/js/jquery-1.4.2.js