Merge branch 'master' of https://github.com/CIRCL/AIL-framework

README.md

@@ -23,11 +23,16 @@ sudo apt-get install screen
 
 You need to create a variable AILENV that will be the installation path:
 
-``export AILENV="/home/user/ail"``
+``export AILENV="/home/user/AIL-framework"``
+
+Usually the installation path is where the project is cloned.
 
 Then create a Python virtual environment:
 
-``virtualenv AILENV``
+```
+cd $AILENV
+virtualenv AILENV
+```
 
 And install these few more packets:
 ```
@@ -35,6 +40,9 @@ sudo apt-get install g++
 sudo apt-get install python-dev
 sudo apt-get install python-tk
 sudo apt-get install screen
+sudo apt-get install libssl-dev
+sudo apt-get install libfreetype6-dev
+sudo apt-get install python-numpy
 ```
 
 Then these modules need to be install with pip inside the virtual environment:
@@ -54,17 +62,18 @@ That's all the packages you can install with pip:
 
 ```
 pip install redis
+pip install logbook
 pip install networkx
 pip install crcmod
 pip install mmh3
 pip install dnspython
 pip install pyzmq
-pip install texttable ----- Queues Monitoring (Optional)
+pip install texttable
 pip install -U textblob
 python -m textblob.download_corpora
 pip install python-magic
 pip install numpy
-pip install flask ----- (Optional)
+pip install flask
 pip install nltk
 pip install matplotlib ----- (sudo ln -s freetype2/ft2build.h in /usr/include/)
 pip install pybloomfiltermmap ----- (you may need to sudo apt-get install libssl-dev)
@@ -115,6 +124,22 @@ cd bin
 ./LAUNCH.sh
 ```
 
+To start with the web interface, you need to fetch the required Javascript/CSS files:
+
+```
+cd $AILENV
+cd var/www/
+bash update_thirdparty.sh
+```
+
+and then you can start the web interface:
+
+```
+cd $AILENV
+cd var/www/
+Flask_server.py
+```
+
 Then you can browse the status of the AIL framework at the following URL:
 
         ``http://localhost:7000/``

bin/ZMQ_Sub_Urls.py

@@ -59,44 +59,47 @@ def main():
     url_regex = "(http|https|ftp)\://([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.(com|edu|gov|int|mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2}))(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*"
 
     while True:
-        if message != None:
+        try:
-            channel, filename, word, score  = message.split()
+            if message != None:
+                channel, filename, word, score  = message.split()
 
-            if prec_filename == None or filename != prec_filename:
+                if prec_filename == None or filename != prec_filename:
-                domains_list = []
+                    domains_list = []
-                PST = P.Paste(filename)
+                    PST = P.Paste(filename)
 
-                for x in PST.get_regex(url_regex):
+                    for x in PST.get_regex(url_regex):
-                    scheme, credential, subdomain, domain, host, tld, port, resource_path, query_string, f1, f2, f3, f4 = x
+                        scheme, credential, subdomain, domain, host, tld, port, resource_path, query_string, f1, f2, f3, f4 = x
-                    domains_list.append(domain)
+                        domains_list.append(domain)
-                    msg = pubchannel + " " + str(x)
+                        msg = pubchannel + " " + str(x)
-                    Pub.send_message(msg)
+                        Pub.send_message(msg)
-                    publisher.debug('{0} Published'.format(x))
+                        publisher.debug('{0} Published'.format(x))
 
-                    if f1 == "onion":
+                        if f1 == "onion":
-                        print domain
+                            print domain
 
-                A_values = lib_refine.checking_A_record(r_serv2, domains_list)
+                    A_values = lib_refine.checking_A_record(r_serv2, domains_list)
 
-                if A_values[0] >= 1:
+                    if A_values[0] >= 1:
-                    PST.__setattr__(channel, A_values)
+                        PST.__setattr__(channel, A_values)
-                    PST.save_attribute_redis(r_serv1, channel, (A_values[0],list(A_values[1])))
+                        PST.save_attribute_redis(r_serv1, channel, (A_values[0],list(A_values[1])))
 
-                    pprint.pprint(A_values)
+                        pprint.pprint(A_values)
-                    publisher.info('{0};{1};{2};{3};{4}'.format("Url", PST.p_source, PST.p_date, PST.p_name, str(A_values[0])+" Valid url detected" ))
+                        publisher.info('{0};{1};{2};{3};{4}'.format("Url", PST.p_source, PST.p_date, PST.p_name, str(A_values[0])+" Valid url detected" ))
-            prec_filename = filename
+                prec_filename = filename
 
-        else:
+            else:
-            if r_serv.sismember("SHUTDOWN_FLAGS", "Urls"):
+                if r_serv.sismember("SHUTDOWN_FLAGS", "Urls"):
-                r_serv.srem("SHUTDOWN_FLAGS", "Urls")
+                    r_serv.srem("SHUTDOWN_FLAGS", "Urls")
-                print "Shutdown Flag Up: Terminating"
+                    print "Shutdown Flag Up: Terminating"
-                publisher.warning("Shutdown Flag Up: Terminating.")
+                    publisher.warning("Shutdown Flag Up: Terminating.")
-                break
+                    break
-            publisher.debug("Script url is Idling 10s")
+                publisher.debug("Script url is Idling 10s")
-            time.sleep(10)
+                time.sleep(10)
-
-        message = Sub.get_msg_from_queue(r_serv)
 
+            message = Sub.get_msg_from_queue(r_serv)
+        except dns.exception.Timeout:
+            print "dns.exception.Timeout"
+            pass
 
 if __name__ == "__main__":
     main()

bin/packages/Paste.py

@@ -16,7 +16,7 @@ Conditions to fulfill to be able to use this class correctly:
 
 """
 
-import os, magic, gzip, langid, pprint, redis, operator, string, re, json
+import os, magic, gzip, langid, pprint, redis, operator, string, re, json, ConfigParser
 from Date import Date
 from Hash import Hash
 
@@ -46,12 +46,22 @@ class Paste(object):
     """
 
     def __init__(self, p_path):
+
+        configfile = './packages/config.cfg'
+        cfg = ConfigParser.ConfigParser()
+        cfg.read(configfile)
+
         self.p_path = p_path
 
         self.p_name = self.p_path.split('/')[-1]
 
         self.p_size = round(os.path.getsize(self.p_path)/1024.0,2)
 
+        self.cache = redis.StrictRedis(
+            host = cfg.get("Redis_Queues", "host"),
+            port = cfg.getint("Redis_Queues", "port"),
+            db = cfg.getint("Redis_Queues", "db"))
+
         self.p_mime = magic.from_buffer(self.get_p_content(), mime = True)
 
         self.p_encoding = None
@@ -82,8 +92,15 @@ class Paste(object):
         PST.get_p_content()
 
         """
-        with gzip.open(self.p_path, 'rb') as F:
+        r_serv = self.cache
-            return F.read()
+
+        paste = r_serv.get(self.p_path)
+        if paste is None:
+            with gzip.open(self.p_path, 'rb') as F:
+		paste = F.read()
+		r_serv.set(self.p_path, paste)
+                r_serv.expire(self.p_path, 300)
+        return paste
 
     def get_lines_info(self):
         """

var/www/Flask_server.py

@@ -63,4 +63,4 @@ def wordstrending():
 
 
 if __name__ == "__main__":
-    app.run(host='172.16.100.123' ,port=7000, threaded=True)
+    app.run(host='0.0.0.0' ,port=7000, threaded=True)