CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'master' of https://github.com/CIRCL/AIL-framework

pull/8/head
Starow 2014-08-11 09:21:09 +02:00
parent a5c1d59d29 54091a2174
commit 192074e569
4 changed files with 83 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
README.md
View File

@ -23,11 +23,16 @@ sudo apt-get install screen
You need to create a variable AILENV that will be the installation path:
``export AILENV="/home/user/ail"``
``export AILENV="/home/user/AIL-framework"``
Usually the installation path is where the project is cloned.
Then create a Python virtual environment:
``virtualenv AILENV``
```
cd $AILENV
virtualenv AILENV
```
And install these few more packets:
```
@ -35,6 +40,9 @@ sudo apt-get install g++
sudo apt-get install python-dev
sudo apt-get install python-tk
sudo apt-get install screen
sudo apt-get install libssl-dev
sudo apt-get install libfreetype6-dev
sudo apt-get install python-numpy
```
Then these modules need to be install with pip inside the virtual environment:
@ -54,17 +62,18 @@ That's all the packages you can install with pip:
```
pip install redis
pip install logbook
pip install networkx
pip install crcmod
pip install mmh3
pip install dnspython
pip install pyzmq
pip install texttable ----- Queues Monitoring (Optional)
pip install texttable
pip install -U textblob
python -m textblob.download_corpora
pip install python-magic
pip install numpy
pip install flask ----- (Optional)
pip install flask
pip install nltk
pip install matplotlib ----- (sudo ln -s freetype2/ft2build.h in /usr/include/)
pip install pybloomfiltermmap ----- (you may need to sudo apt-get install libssl-dev)
@ -115,6 +124,22 @@ cd bin
./LAUNCH.sh
```
To start with the web interface, you need to fetch the required Javascript/CSS files:
```
cd $AILENV
cd var/www/
bash update_thirdparty.sh
```
and then you can start the web interface:
```
cd $AILENV
cd var/www/
Flask_server.py
```
Then you can browse the status of the AIL framework at the following URL:
``http://localhost:7000/``

63
bin/ZMQ_Sub_Urls.py
View File

@ -59,44 +59,47 @@ def main():
url_regex = "(http|https|ftp)\://([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.(com|edu|gov|int|mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2}))(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*"
while True:
if message != None:
channel, filename, word, score = message.split()
try:
if message != None:
channel, filename, word, score = message.split()
if prec_filename == None or filename != prec_filename:
domains_list = []
PST = P.Paste(filename)
if prec_filename == None or filename != prec_filename:
domains_list = []
PST = P.Paste(filename)
for x in PST.get_regex(url_regex):
scheme, credential, subdomain, domain, host, tld, port, resource_path, query_string, f1, f2, f3, f4 = x
domains_list.append(domain)
msg = pubchannel + " " + str(x)
Pub.send_message(msg)
publisher.debug('{0} Published'.format(x))
for x in PST.get_regex(url_regex):
scheme, credential, subdomain, domain, host, tld, port, resource_path, query_string, f1, f2, f3, f4 = x
domains_list.append(domain)
msg = pubchannel + " " + str(x)
Pub.send_message(msg)
publisher.debug('{0} Published'.format(x))
if f1 == "onion":
print domain
if f1 == "onion":
print domain
A_values = lib_refine.checking_A_record(r_serv2, domains_list)
A_values = lib_refine.checking_A_record(r_serv2, domains_list)
if A_values[0] >= 1:
PST.__setattr__(channel, A_values)
PST.save_attribute_redis(r_serv1, channel, (A_values[0],list(A_values[1])))
if A_values[0] >= 1:
PST.__setattr__(channel, A_values)
PST.save_attribute_redis(r_serv1, channel, (A_values[0],list(A_values[1])))
pprint.pprint(A_values)
publisher.info('{0};{1};{2};{3};{4}'.format("Url", PST.p_source, PST.p_date, PST.p_name, str(A_values[0])+" Valid url detected" ))
prec_filename = filename
pprint.pprint(A_values)
publisher.info('{0};{1};{2};{3};{4}'.format("Url", PST.p_source, PST.p_date, PST.p_name, str(A_values[0])+" Valid url detected" ))
prec_filename = filename
else:
if r_serv.sismember("SHUTDOWN_FLAGS", "Urls"):
r_serv.srem("SHUTDOWN_FLAGS", "Urls")
print "Shutdown Flag Up: Terminating"
publisher.warning("Shutdown Flag Up: Terminating.")
break
publisher.debug("Script url is Idling 10s")
time.sleep(10)
message = Sub.get_msg_from_queue(r_serv)
else:
if r_serv.sismember("SHUTDOWN_FLAGS", "Urls"):
r_serv.srem("SHUTDOWN_FLAGS", "Urls")
print "Shutdown Flag Up: Terminating"
publisher.warning("Shutdown Flag Up: Terminating.")
break
publisher.debug("Script url is Idling 10s")
time.sleep(10)
message = Sub.get_msg_from_queue(r_serv)
except dns.exception.Timeout:
print "dns.exception.Timeout"
pass
if __name__ == "__main__":
main()

23
bin/packages/Paste.py
View File

@ -16,7 +16,7 @@ Conditions to fulfill to be able to use this class correctly:
"""
import os, magic, gzip, langid, pprint, redis, operator, string, re, json
import os, magic, gzip, langid, pprint, redis, operator, string, re, json, ConfigParser
from Date import Date
from Hash import Hash
@ -46,12 +46,22 @@ class Paste(object):
"""
def __init__(self, p_path):
configfile = './packages/config.cfg'
cfg = ConfigParser.ConfigParser()
cfg.read(configfile)
self.p_path = p_path
self.p_name = self.p_path.split('/')[-1]
self.p_size = round(os.path.getsize(self.p_path)/1024.0,2)
self.cache = redis.StrictRedis(
host = cfg.get("Redis_Queues", "host"),
port = cfg.getint("Redis_Queues", "port"),
db = cfg.getint("Redis_Queues", "db"))
self.p_mime = magic.from_buffer(self.get_p_content(), mime = True)
self.p_encoding = None
@ -82,8 +92,15 @@ class Paste(object):
PST.get_p_content()
"""
with gzip.open(self.p_path, 'rb') as F:
return F.read()
r_serv = self.cache
paste = r_serv.get(self.p_path)
if paste is None:
with gzip.open(self.p_path, 'rb') as F:
paste = F.read()
r_serv.set(self.p_path, paste)
r_serv.expire(self.p_path, 300)
return paste
def get_lines_info(self):
"""

2
var/www/Flask_server.py
View File

@ -63,4 +63,4 @@ def wordstrending():
if __name__ == "__main__":
app.run(host='172.16.100.123' ,port=7000, threaded=True)
app.run(host='0.0.0.0' ,port=7000, threaded=True)