From a80f6ca0c2205fec70f9d02310619c4d8dc123e1 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Tue, 9 Aug 2016 11:59:36 +0200 Subject: [PATCH] Bug fix related with redis: Fixed typo key in redis for module creditcard and sqlinjection Modified Curve redisLvlDb server Modified Url.py so that it forwards name of protocol from saved protocolsfile Added Cache control in Flask Modified key-tab name into keys-tab --- bin/CreditCard.py | 2 +- bin/Curve.py | 6 +++--- bin/LAUNCH.sh | 5 +++++ bin/SQLInjectionDetection.py | 2 +- bin/Url.py | 14 +++++++++++++- bin/WebStats.py | 2 +- files/protocolsfile | 2 +- var/www/Flask_server.py | 12 ++++++++++-- var/www/static/js/plot-graph.js | 10 ++++++---- var/www/templates/Trending.html | 4 ++++ var/www/templates/browse_important_paste.html | 5 ++--- 11 files changed, 47 insertions(+), 17 deletions(-) diff --git a/bin/CreditCard.py b/bin/CreditCard.py index e5f8020e..42eefd09 100755 --- a/bin/CreditCard.py +++ b/bin/CreditCard.py @@ -68,7 +68,7 @@ if __name__ == "__main__": #Send to duplicate p.populate_set_out(filepath, 'Redis_Duplicate') #send to Browse_warning_paste - p.populate_set_out('creditCard;{}'.format(filename), 'BrowseWarningPaste') + p.populate_set_out('creditcard;{}'.format(filename), 'BrowseWarningPaste') else: publisher.info('{}CreditCard related'.format(to_print)) else: diff --git a/bin/Curve.py b/bin/Curve.py index 4a627483..4c3378ea 100755 --- a/bin/Curve.py +++ b/bin/Curve.py @@ -40,9 +40,9 @@ if __name__ == "__main__": # REDIS # r_serv1 = redis.StrictRedis( - host=p.config.get("Redis_Level_DB", "host"), - port=p.config.get("Redis_Level_DB", "port"), - db=p.config.get("Redis_Level_DB", "db")) + host=p.config.get("Redis_Level_DB_Curve", "host"), + port=p.config.get("Redis_Level_DB_Curve", "port"), + db=p.config.get("Redis_Level_DB_Curve", "db")) # FUNCTIONS # publisher.info("Script Curve started") diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 86877c2a..7f1220c9 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -67,6 +67,7 @@ function launching_lvldb { db1_y='2013' db2_y='2014' db3_y='2016' + db4_y='3016' nb_db=13 screen -dmS "LevelDB" @@ -78,6 +79,10 @@ function launching_lvldb { screen -S "LevelDB" -X screen -t "2014" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2014/ -P '$db2_y' -M '$nb_db'; read x' sleep 0.1 screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db3_y' -M '$nb_db'; read x' + + # For Curve + sleep 0.1 + screen -S "LevelDB" -X screen -t "3016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'3016/ -P '$db4_y' -M '$nb_db'; read x' } function launching_logs { diff --git a/bin/SQLInjectionDetection.py b/bin/SQLInjectionDetection.py index d6c3efa5..de3d4c90 100755 --- a/bin/SQLInjectionDetection.py +++ b/bin/SQLInjectionDetection.py @@ -77,7 +77,7 @@ def analyse(url, path): #Send to duplicate p.populate_set_out(path, 'Duplicate') #send to Browse_warning_paste - p.populate_set_out('sqlInjectionDetection;{}'.format(path), 'BrowseWarningPaste') + p.populate_set_out('sqlinjection;{}'.format(path), 'BrowseWarningPaste') else: print "Potential SQL injection:" print urllib2.unquote(url) diff --git a/bin/Url.py b/bin/Url.py index 398ca49d..9892697c 100755 --- a/bin/Url.py +++ b/bin/Url.py @@ -3,6 +3,7 @@ import redis import pprint import time +import os import dns.exception from packages import Paste from packages import lib_refine @@ -39,6 +40,10 @@ if __name__ == "__main__": port=p.config.getint("Redis_Cache", "port"), db=p.config.getint("Redis_Cache", "db")) + # Protocol file path + protocolsfile_path = os.path.join(os.environ['AIL_HOME'], + p.config.get("Directories", "protocolsfile")) + # Country to log as critical cc_critical = p.config.get("Url", "cc_critical") @@ -52,7 +57,14 @@ if __name__ == "__main__": prec_filename = None faup = Faup() - url_regex = "(http|https|ftp)\://([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.(com|edu|gov|int|mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2}))(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*" + # Get all uri from protocolsfile (Used for Curve) + uri_scheme = "" + with open(protocolsfile_path, 'r') as scheme_file: + for scheme in scheme_file: + uri_scheme += scheme[:-1]+"|" + uri_scheme = uri_scheme[:-1] + + url_regex = "("+uri_scheme+")\://([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.(com|edu|gov|int|mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2}))(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*" while True: if message is not None: diff --git a/bin/WebStats.py b/bin/WebStats.py index c6507542..837fa2fe 100755 --- a/bin/WebStats.py +++ b/bin/WebStats.py @@ -26,6 +26,7 @@ def analyse(server, field_name, date, url_parsed): prev_score = server.hget(field, date) if prev_score is not None: server.hset(field, date, int(prev_score) + 1) + else: server.hset(field, date, 1) if field_name == "domain": #save domain in a set for the monthly plot @@ -67,7 +68,6 @@ def compute_progression(server, field_name, num_day, url_parsed): divisor = value_list_reversed[i-1] if value_list_reversed[i-1] != 0 else 1 keyword_increase += value_list_reversed[i] / divisor - # filter if (keyword_total_sum > threshold_total_sum) and (keyword_increase > threshold_increase): diff --git a/files/protocolsfile b/files/protocolsfile index ffece4b6..61d0c127 100644 --- a/files/protocolsfile +++ b/files/protocolsfile @@ -96,5 +96,5 @@ rtmfp ipps pkcs11 acct -example +example vnc diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 9b5d4679..22d407f8 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -164,7 +164,6 @@ def showpaste(content_range): def getPastebyType(server, module_name): all_path = [] for path in server.smembers('WARNING_'+module_name): - #all_path.append("/home/mokaddem/AIL-framework/PASTES/archive/paste.debian.net/2016/06/30/771058.gz") all_path.append(path) return all_path @@ -198,7 +197,16 @@ def get_top_relevant_data(server, module_name): member_set.insert(0, ("passed_days", days)) return member_set - +# ========= CACHE CONTROL ======== +@app.after_request +def add_header(response): + """ + Add headers to both force latest IE rendering engine or Chrome Frame, + and also to cache the rendered page for 10 minutes. + """ + response.headers['X-UA-Compatible'] = 'IE=Edge,chrome=1' + response.headers['Cache-Control'] = 'public, max-age=0' + return response # ============ ROUTES ============ diff --git a/var/www/static/js/plot-graph.js b/var/www/static/js/plot-graph.js index f9a2ebf7..6bbd4cb9 100644 --- a/var/www/static/js/plot-graph.js +++ b/var/www/static/js/plot-graph.js @@ -112,19 +112,21 @@ function Graph(id_pannel, path, header_size){ // display the top headers function setVis(max_display){ - headings = this.graph.getLabels(); + var headings = this.graph.getLabels(); headings.splice(0,1); var sorted_list = new Array(); - today = new Date().getDate()-1; // Take the top from yesterday so that we can see the current evolution + var today = new Date().getDate(); + //today = new Date().getDate()-1; // Could take the top from yesterday so that we can see the current evolution for( i=0; iCredit cards
  • SQL injections
  • CVEs
    • -
  • Keys
    • +
  • Keys
  • Mails
  • Phones
    • @@ -119,7 +119,7 @@
    -
    +
    @@ -150,7 +150,6 @@ var previous_tab = $('[data-attribute-name="credential'); var loading_gif = ""; $('.nav-tabs a').on('shown.bs.tab', function(event){ - console.log(event.target); var dataPath = $(event.target).attr('data-attribute-name'); $.get("{{ url_for('importantPasteByModule') }}"+"?moduleName="+dataPath, function(data, status){ var currentTab = $('[name].active').children();