diff --git a/bin/CreditCard.py b/bin/CreditCard.py index e5f8020e..42eefd09 100755 --- a/bin/CreditCard.py +++ b/bin/CreditCard.py @@ -68,7 +68,7 @@ if __name__ == "__main__": #Send to duplicate p.populate_set_out(filepath, 'Redis_Duplicate') #send to Browse_warning_paste - p.populate_set_out('creditCard;{}'.format(filename), 'BrowseWarningPaste') + p.populate_set_out('creditcard;{}'.format(filename), 'BrowseWarningPaste') else: publisher.info('{}CreditCard related'.format(to_print)) else: diff --git a/bin/Curve.py b/bin/Curve.py index 4a627483..4c3378ea 100755 --- a/bin/Curve.py +++ b/bin/Curve.py @@ -40,9 +40,9 @@ if __name__ == "__main__": # REDIS # r_serv1 = redis.StrictRedis( - host=p.config.get("Redis_Level_DB", "host"), - port=p.config.get("Redis_Level_DB", "port"), - db=p.config.get("Redis_Level_DB", "db")) + host=p.config.get("Redis_Level_DB_Curve", "host"), + port=p.config.get("Redis_Level_DB_Curve", "port"), + db=p.config.get("Redis_Level_DB_Curve", "db")) # FUNCTIONS # publisher.info("Script Curve started") diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 86877c2a..7f1220c9 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -67,6 +67,7 @@ function launching_lvldb { db1_y='2013' db2_y='2014' db3_y='2016' + db4_y='3016' nb_db=13 screen -dmS "LevelDB" @@ -78,6 +79,10 @@ function launching_lvldb { screen -S "LevelDB" -X screen -t "2014" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2014/ -P '$db2_y' -M '$nb_db'; read x' sleep 0.1 screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db3_y' -M '$nb_db'; read x' + + # For Curve + sleep 0.1 + screen -S "LevelDB" -X screen -t "3016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'3016/ -P '$db4_y' -M '$nb_db'; read x' } function launching_logs { diff --git a/bin/SQLInjectionDetection.py b/bin/SQLInjectionDetection.py index d6c3efa5..de3d4c90 100755 --- a/bin/SQLInjectionDetection.py +++ b/bin/SQLInjectionDetection.py @@ -77,7 +77,7 @@ def analyse(url, path): #Send to duplicate p.populate_set_out(path, 'Duplicate') #send to Browse_warning_paste - p.populate_set_out('sqlInjectionDetection;{}'.format(path), 'BrowseWarningPaste') + p.populate_set_out('sqlinjection;{}'.format(path), 'BrowseWarningPaste') else: print "Potential SQL injection:" print urllib2.unquote(url) diff --git a/bin/Url.py b/bin/Url.py index 398ca49d..9892697c 100755 --- a/bin/Url.py +++ b/bin/Url.py @@ -3,6 +3,7 @@ import redis import pprint import time +import os import dns.exception from packages import Paste from packages import lib_refine @@ -39,6 +40,10 @@ if __name__ == "__main__": port=p.config.getint("Redis_Cache", "port"), db=p.config.getint("Redis_Cache", "db")) + # Protocol file path + protocolsfile_path = os.path.join(os.environ['AIL_HOME'], + p.config.get("Directories", "protocolsfile")) + # Country to log as critical cc_critical = p.config.get("Url", "cc_critical") @@ -52,7 +57,14 @@ if __name__ == "__main__": prec_filename = None faup = Faup() - url_regex = "(http|https|ftp)\://([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.(com|edu|gov|int|mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2}))(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*" + # Get all uri from protocolsfile (Used for Curve) + uri_scheme = "" + with open(protocolsfile_path, 'r') as scheme_file: + for scheme in scheme_file: + uri_scheme += scheme[:-1]+"|" + uri_scheme = uri_scheme[:-1] + + url_regex = "("+uri_scheme+")\://([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.(com|edu|gov|int|mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2}))(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*" while True: if message is not None: diff --git a/bin/WebStats.py b/bin/WebStats.py index c6507542..837fa2fe 100755 --- a/bin/WebStats.py +++ b/bin/WebStats.py @@ -26,6 +26,7 @@ def analyse(server, field_name, date, url_parsed): prev_score = server.hget(field, date) if prev_score is not None: server.hset(field, date, int(prev_score) + 1) + else: server.hset(field, date, 1) if field_name == "domain": #save domain in a set for the monthly plot @@ -67,7 +68,6 @@ def compute_progression(server, field_name, num_day, url_parsed): divisor = value_list_reversed[i-1] if value_list_reversed[i-1] != 0 else 1 keyword_increase += value_list_reversed[i] / divisor - # filter if (keyword_total_sum > threshold_total_sum) and (keyword_increase > threshold_increase): diff --git a/files/protocolsfile b/files/protocolsfile index ffece4b6..61d0c127 100644 --- a/files/protocolsfile +++ b/files/protocolsfile @@ -96,5 +96,5 @@ rtmfp ipps pkcs11 acct -example +example vnc diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 9b5d4679..22d407f8 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -164,7 +164,6 @@ def showpaste(content_range): def getPastebyType(server, module_name): all_path = [] for path in server.smembers('WARNING_'+module_name): - #all_path.append("/home/mokaddem/AIL-framework/PASTES/archive/paste.debian.net/2016/06/30/771058.gz") all_path.append(path) return all_path @@ -198,7 +197,16 @@ def get_top_relevant_data(server, module_name): member_set.insert(0, ("passed_days", days)) return member_set - +# ========= CACHE CONTROL ======== +@app.after_request +def add_header(response): + """ + Add headers to both force latest IE rendering engine or Chrome Frame, + and also to cache the rendered page for 10 minutes. + """ + response.headers['X-UA-Compatible'] = 'IE=Edge,chrome=1' + response.headers['Cache-Control'] = 'public, max-age=0' + return response # ============ ROUTES ============ diff --git a/var/www/static/js/plot-graph.js b/var/www/static/js/plot-graph.js index f9a2ebf7..6bbd4cb9 100644 --- a/var/www/static/js/plot-graph.js +++ b/var/www/static/js/plot-graph.js @@ -112,19 +112,21 @@ function Graph(id_pannel, path, header_size){ // display the top headers function setVis(max_display){ - headings = this.graph.getLabels(); + var headings = this.graph.getLabels(); headings.splice(0,1); var sorted_list = new Array(); - today = new Date().getDate()-1; // Take the top from yesterday so that we can see the current evolution + var today = new Date().getDate(); + //today = new Date().getDate()-1; // Could take the top from yesterday so that we can see the current evolution for( i=0; iCredit cards
  • SQL injections
  • CVEs
    • -
  • Keys
    • +
  • Keys
  • Mails
  • Phones
    • @@ -119,7 +119,7 @@
    -
    +
    @@ -150,7 +150,6 @@ var previous_tab = $('[data-attribute-name="credential'); var loading_gif = ""; $('.nav-tabs a').on('shown.bs.tab', function(event){ - console.log(event.target); var dataPath = $(event.target).attr('data-attribute-name'); $.get("{{ url_for('importantPasteByModule') }}"+"?moduleName="+dataPath, function(data, status){ var currentTab = $('[name].active').children();