Merge branch 'master' of https://github.com/CIRCL/AIL-framework

README.md

@@ -23,11 +23,16 @@ sudo apt-get install screen
 
 You need to create a variable AILENV that will be the installation path:
 
-``export AILENV="/home/user/ail"``
+``export AILENV="/home/user/AIL-framework"``
+
+Usually the installation path is where the project is cloned.
 
 Then create a Python virtual environment:
 
-``virtualenv AILENV``
+```
+cd $AILENV
+virtualenv AILENV
+```
 
 And install these few more packets:
 ```
@@ -35,6 +40,9 @@ sudo apt-get install g++
 sudo apt-get install python-dev
 sudo apt-get install python-tk
 sudo apt-get install screen
+sudo apt-get install libssl-dev
+sudo apt-get install libfreetype6-dev
+sudo apt-get install python-numpy
 ```
 
 Then these modules need to be install with pip inside the virtual environment:
@@ -54,17 +62,18 @@ That's all the packages you can install with pip:
 
 ```
 pip install redis
+pip install logbook
 pip install networkx
 pip install crcmod
 pip install mmh3
 pip install dnspython
 pip install pyzmq
-pip install texttable ----- Queues Monitoring (Optional)
+pip install texttable
 pip install -U textblob
 python -m textblob.download_corpora
 pip install python-magic
 pip install numpy
-pip install flask ----- (Optional)
+pip install flask
 pip install nltk
 pip install matplotlib ----- (sudo ln -s freetype2/ft2build.h in /usr/include/)
 pip install pybloomfiltermmap ----- (you may need to sudo apt-get install libssl-dev)
@@ -115,6 +124,22 @@ cd bin
 ./LAUNCH.sh
 ```
 
+To start with the web interface, you need to fetch the required Javascript/CSS files:
+
+```
+cd $AILENV
+cd var/www/
+bash update_thirdparty.sh
+```
+
+and then you can start the web interface:
+
+```
+cd $AILENV
+cd var/www/
+Flask_server.py
+```
+
 Then you can browse the status of the AIL framework at the following URL:
 
         ``http://localhost:7000/``

bin/ZMQ_Sub_Urls.py

@@ -59,6 +59,7 @@ def main():
     url_regex = "(http|https|ftp)\://([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.(com|edu|gov|int|mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2}))(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*"
 
     while True:
+        try:
             if message != None:
                 channel, filename, word, score  = message.split()
 
@@ -96,7 +97,9 @@ def main():
                 time.sleep(10)
 
             message = Sub.get_msg_from_queue(r_serv)
-
+        except dns.exception.Timeout:
+            print "dns.exception.Timeout"
+            pass
 
 if __name__ == "__main__":
     main()

bin/packages/Paste.py

@@ -16,7 +16,7 @@ Conditions to fulfill to be able to use this class correctly:
 
 """
 
-import os, magic, gzip, langid, pprint, redis, operator, string, re, json
+import os, magic, gzip, langid, pprint, redis, operator, string, re, json, ConfigParser
 from Date import Date
 from Hash import Hash
 
@@ -46,12 +46,22 @@ class Paste(object):
     """
 
     def __init__(self, p_path):
+
+        configfile = './packages/config.cfg'
+        cfg = ConfigParser.ConfigParser()
+        cfg.read(configfile)
+
         self.p_path = p_path
 
         self.p_name = self.p_path.split('/')[-1]
 
         self.p_size = round(os.path.getsize(self.p_path)/1024.0,2)
 
+        self.cache = redis.StrictRedis(
+            host = cfg.get("Redis_Queues", "host"),
+            port = cfg.getint("Redis_Queues", "port"),
+            db = cfg.getint("Redis_Queues", "db"))
+
         self.p_mime = magic.from_buffer(self.get_p_content(), mime = True)
 
         self.p_encoding = None
@@ -82,8 +92,15 @@ class Paste(object):
         PST.get_p_content()
 
         """
+        r_serv = self.cache
+
+        paste = r_serv.get(self.p_path)
+        if paste is None:
             with gzip.open(self.p_path, 'rb') as F:
-            return F.read()
+		paste = F.read()
+		r_serv.set(self.p_path, paste)
+                r_serv.expire(self.p_path, 300)
+        return paste
 
     def get_lines_info(self):
         """

var/www/Flask_server.py

@@ -63,4 +63,4 @@ def wordstrending():
 
 
 if __name__ == "__main__":
-    app.run(host='172.16.100.123' ,port=7000, threaded=True)
+    app.run(host='0.0.0.0' ,port=7000, threaded=True)