diff --git a/.gitignore b/.gitignore index 88b623fa..46d87d71 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,9 @@ logs/ var/www/static/ !var/www/static/css/dygraph_gallery.css !var/www/static/js/indexjavascript.js +!var/www/static/js/moduleTrending.js +!var/www/static/js/plot-graph.js +!var/www/static/js/trendingchart.js # Local config bin/packages/config.cfg diff --git a/bin/Browse_warning_paste.py b/bin/Browse_warning_paste.py new file mode 100755 index 00000000..49444979 --- /dev/null +++ b/bin/Browse_warning_paste.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python2 +# -*-coding:UTF-8 -* + +""" +The Browse_warning_paste module +==================== + +This module saved signaled paste (logged as 'warning') in redis for further usage +like browsing by category + +Its input comes from other modules, namely: + Credential, CreditCard, SQLinjection, CVE, Keys, Mail and Phone + +""" + +import redis +import time +from datetime import datetime, timedelta +from packages import Paste +from pubsublogger import publisher +from Helper import Process + +if __name__ == "__main__": + publisher.port = 6380 + publisher.channel = "Script" + + config_section = 'Browse_warning_paste' + + p = Process(config_section) + + server = redis.StrictRedis( + host=p.config.get("Redis_Level_DB", "host"), + port=p.config.get("Redis_Level_DB", "port"), + db=p.config.get("Redis_Level_DB", "db")) + + # FUNCTIONS # + publisher.info("Script duplicate started") + + while True: + message = p.get_from_set() + if message is not None: + module_name, p_path = message.split(';') + #PST = Paste.Paste(p_path) + else: + publisher.debug("Script Attribute is idling 10s") + time.sleep(10) + continue + + # Add in redis + # Format in set: WARNING_moduleName -> p_path + key = "WARNING_" + module_name + print key + ' -> ' + p_path + server.sadd(key, p_path) + + publisher.info('Saved in warning paste {}'.format(p_path)) + #print 'Saved in warning paste {}'.format(p_path) + diff --git a/bin/Credential.py b/bin/Credential.py index d81c9ff6..d8bb8a84 100755 --- a/bin/Credential.py +++ b/bin/Credential.py @@ -1,10 +1,12 @@ #!/usr/bin/env python2 # -*-coding:UTF-8 -* import time +import sys from packages import Paste from pubsublogger import publisher from Helper import Process import re +from pyfaup.faup import Faup if __name__ == "__main__": publisher.port = 6380 @@ -13,15 +15,17 @@ if __name__ == "__main__": p = Process(config_section) publisher.info("Find credentials") + faup = Faup() + critical = 8 regex_web = "((?:https?:\/\/)[-_0-9a-zA-Z]+\.[0-9a-zA-Z]+)" regex_cred = "[a-zA-Z0-9._-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}:[a-zA-Z0-9\_\-]+" + regex_site_for_stats = "@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}:" while True: message = p.get_from_set() if message is None: publisher.debug("Script Credential is Idling 10s") - print('Sleeping') time.sleep(10) continue @@ -37,11 +41,12 @@ if __name__ == "__main__": if len(creds) == 0: continue - sites = set(re.findall(regex_web, content)) + sites= re.findall(regex_web, content) #Use to count occurences + sites_set = set(re.findall(regex_web, content)) message = 'Checked {} credentials found.'.format(len(creds)) - if sites: - message += ' Related websites: {}'.format(', '.join(sites)) + if sites_set: + message += ' Related websites: {}'.format(', '.join(sites_set)) to_print = 'Credential;{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, message) @@ -51,8 +56,33 @@ if __name__ == "__main__": print("========> Found more than 10 credentials in this file : {}".format(filepath)) publisher.warning(to_print) #Send to duplicate - p.populate_set_out(filepath) - if sites: - print("=======> Probably on : {}".format(', '.join(sites))) + p.populate_set_out(filepath, 'Duplicate') + #Send to BrowseWarningPaste + p.populate_set_out('credential;{}'.format(filepath), 'BrowseWarningPaste') + + #Put in form, count occurences, then send to moduleStats + creds_sites = {} + site_occurence = re.findall(regex_site_for_stats, content) + for site in site_occurence: + site_domain = site[1:-1] + if site_domain in creds_sites.keys(): + creds_sites[site_domain] += 1 + else: + creds_sites[site_domain] = 1 + + for url in sites: + faup.decode(url) + domain = faup.get()['domain'] + if domain in creds_sites.keys(): + creds_sites[domain] += 1 + else: + creds_sites[domain] = 1 + + for site, num in creds_sites.iteritems(): # Send for each different site to moduleStats + print 'credential;{};{};{}'.format(num, site, paste.p_date) + p.populate_set_out('credential;{};{};{}'.format(num, site, paste.p_date), 'ModuleStats') + + if sites_set: + print("=======> Probably on : {}".format(', '.join(sites_set))) else: publisher.info(to_print) diff --git a/bin/CreditCard.py b/bin/CreditCard.py index 18703f4e..42eefd09 100755 --- a/bin/CreditCard.py +++ b/bin/CreditCard.py @@ -66,7 +66,9 @@ if __name__ == "__main__": publisher.warning('{}Checked {} valid number(s)'.format( to_print, len(creditcard_set))) #Send to duplicate - p.populate_set_out(filename) + p.populate_set_out(filepath, 'Redis_Duplicate') + #send to Browse_warning_paste + p.populate_set_out('creditcard;{}'.format(filename), 'BrowseWarningPaste') else: publisher.info('{}CreditCard related'.format(to_print)) else: diff --git a/bin/Curve.py b/bin/Curve.py index 4a627483..4c3378ea 100755 --- a/bin/Curve.py +++ b/bin/Curve.py @@ -40,9 +40,9 @@ if __name__ == "__main__": # REDIS # r_serv1 = redis.StrictRedis( - host=p.config.get("Redis_Level_DB", "host"), - port=p.config.get("Redis_Level_DB", "port"), - db=p.config.get("Redis_Level_DB", "db")) + host=p.config.get("Redis_Level_DB_Curve", "host"), + port=p.config.get("Redis_Level_DB_Curve", "port"), + db=p.config.get("Redis_Level_DB_Curve", "db")) # FUNCTIONS # publisher.info("Script Curve started") diff --git a/bin/Cve.py b/bin/Cve.py index 7323ee5a..97e5aaae 100755 --- a/bin/Cve.py +++ b/bin/Cve.py @@ -25,6 +25,11 @@ def search_cve(message): print('{} contains CVEs'.format(paste.p_name)) publisher.warning('{} contains CVEs'.format(paste.p_name)) + #send to Browse_warning_paste + p.populate_set_out('cve;{}'.format(filepath), 'BrowseWarningPaste') + #Send to duplicate + p.populate_set_out(filepath, 'Duplicate') + if __name__ == '__main__': # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh) # Port of the redis instance used by pubsublogger @@ -53,5 +58,3 @@ if __name__ == '__main__': # Do something with the message from the queue search_cve(message) - # (Optional) Send that thing to the next queue - #p.populate_set_out(something_has_been_done) diff --git a/bin/Duplicate_ssdeep_v2.py b/bin/Duplicate_ssdeep_v2.py index e8930c02..d2efcab3 100755 --- a/bin/Duplicate_ssdeep_v2.py +++ b/bin/Duplicate_ssdeep_v2.py @@ -7,10 +7,12 @@ The Duplicate module This huge module is, in short term, checking duplicates. Its input comes from other modules, namely: - Credential, CreditCard, Keys, Mails and Phone + Credential, CreditCard, Keys, Mails, SQLinjectionDetection, CVE and Phone This one differ from v1 by only using redis and not json file stored on disk +Perform comparisions with ssdeep and tlsh + Requirements: ------------- @@ -22,6 +24,7 @@ import time from datetime import datetime, timedelta import json import ssdeep +import tlsh from packages import Paste from pubsublogger import publisher @@ -36,8 +39,12 @@ if __name__ == "__main__": p = Process(config_section) maximum_month_range = int(p.config.get("Modules_Duplicates", "maximum_month_range")) - threshold_duplicate = int(p.config.get("Modules_Duplicates", "threshold_duplicate")) - min_paste_size = float(p.config.get("Modules_Duplicates", "min_paste_size")) + threshold_duplicate_ssdeep = int(p.config.get("Modules_Duplicates", "threshold_duplicate_ssdeep")) + threshold_duplicate_tlsh = int(p.config.get("Modules_Duplicates", "threshold_duplicate_tlsh")) + threshold_set = {} + threshold_set['ssdeep'] = threshold_duplicate_ssdeep + threshold_set['tlsh'] = threshold_duplicate_tlsh + min_paste_size = float(p.config.get("Modules_Duplicates", "min_paste_size")) # REDIS # dico_redis = {} @@ -47,7 +54,7 @@ if __name__ == "__main__": dico_redis[str(year)+str(month).zfill(2)] = redis.StrictRedis( host=p.config.get("Redis_Level_DB", "host"), port=year, db=month) - #print("dup: "+str(year)+str(month).zfill(2)+"\n") + #print("dup: "+str(year)+str(month).zfill(2)+"\n") # FUNCTIONS # publisher.info("Script duplicate started") @@ -70,10 +77,11 @@ if __name__ == "__main__": continue # the paste is too small - if (PST._get_p_size() < min_paste_size): + if (PST._get_p_size() < min_paste_size): continue PST._set_p_hash_kind("ssdeep") + PST._set_p_hash_kind("tlsh") # Assignate the correct redis connexion r_serv1 = dico_redis[PST.p_date.year + PST.p_date.month] @@ -86,7 +94,7 @@ if __name__ == "__main__": curr_date_range = date_today - timedelta(days = diff_month*30.4166666) to_append = str(curr_date_range.year)+str(curr_date_range.month).zfill(2) dico_range_list.append(to_append) - + # Use all dico in range dico_range_list = dico_range_list[0:maximum_month_range] @@ -95,43 +103,51 @@ if __name__ == "__main__": r_serv0 = dico_redis[yearly_index] r_serv0.incr("current_index") index = r_serv0.get("current_index")+str(PST.p_date) - - # Open selected dico range + + # Open selected dico range opened_dico = [] for dico_name in dico_range_list: opened_dico.append([dico_name, dico_redis[dico_name]]) - + # retrieve hash from paste - paste_hash = PST._get_p_hash() - + paste_hashes = PST._get_p_hash() + # Go throught the Database of the dico (of the month) for curr_dico_name, curr_dico_redis in opened_dico: - for dico_hash in curr_dico_redis.smembers('HASHS'): - try: - percent = ssdeep.compare(dico_hash, paste_hash) - if percent > threshold_duplicate: - # Go throught the Database of the dico filter (month) - r_serv_dico = dico_redis[curr_dico_name] - - # index of paste - index_current = r_serv_dico.get(dico_hash) - paste_path = r_serv_dico.get(index_current) - if paste_path != None: - hash_dico[dico_hash] = (paste_path, percent) + for hash_type, paste_hash in paste_hashes.iteritems(): + for dico_hash in curr_dico_redis.smembers('HASHS_'+hash_type): + try: + if hash_type == 'ssdeep': + percent = 100-ssdeep.compare(dico_hash, paste_hash) + else: + percent = tlsh.diffxlen(dico_hash, paste_hash) - #print 'comparing: ' + str(PST.p_path[44:]) + ' and ' + str(paste_path[44:]) + ' percentage: ' + str(percent) - except: - # ssdeep hash not comparable - print 'ssdeep hash not comparable, cleaning bad hash: '+dico_hash - curr_dico_redis.srem('HASHS', dico_hash) + threshold_duplicate = threshold_set[hash_type] + if percent < threshold_duplicate: + percent = 100 - percent if hash_type == 'ssdeep' else percent #recovert the correct percent value for ssdeep + # Go throught the Database of the dico filter (month) + r_serv_dico = dico_redis[curr_dico_name] + + # index of paste + index_current = r_serv_dico.get(dico_hash) + paste_path = r_serv_dico.get(index_current) + if paste_path != None: + hash_dico[dico_hash] = (hash_type, paste_path, percent) + + print '['+hash_type+'] '+'comparing: ' + str(PST.p_path[44:]) + ' and ' + str(paste_path[44:]) + ' percentage: ' + str(percent) + except Exception,e: + print str(e) + #print 'hash not comparable, bad hash: '+dico_hash+' , current_hash: '+paste_hash # Add paste in DB after checking to prevent its analysis twice - # hash_i -> index_i AND index_i -> PST.PATH + # hash_type_i -> index_i AND index_i -> PST.PATH r_serv1.set(index, PST.p_path) r_serv1.sadd("INDEX", index) - # Adding the hash in Redis - r_serv1.set(paste_hash, index) - r_serv1.sadd("HASHS", paste_hash) + # Adding hashes in Redis + for hash_type, paste_hash in paste_hashes.iteritems(): + r_serv1.set(paste_hash, index) + r_serv1.sadd("HASHS_"+hash_type, paste_hash) + ##################### Similarity found ####################### # if there is data in this dictionnary @@ -153,7 +169,7 @@ if __name__ == "__main__": publisher.debug('{}Processed in {} sec'.format(to_print, y-x)) #print '{}Processed in {} sec'.format(to_print, y-x) - + except IOError: to_print = 'Duplicate;{};{};{};'.format( PST.p_source, PST.p_date, PST.p_name) diff --git a/bin/Global.py b/bin/Global.py index 8b6e482f..9cacbc88 100755 --- a/bin/Global.py +++ b/bin/Global.py @@ -52,7 +52,7 @@ if __name__ == '__main__': else: # TODO Store the name of the empty paste inside a Redis-list. print "Empty Paste: not processed" - publisher.debug("Empty Paste: {0} not processed".format(paste)) + publisher.debug("Empty Paste: {0} not processed".format(message)) continue else: print "Empty Queues: Waiting..." diff --git a/bin/Keys.py b/bin/Keys.py index 9c44f60a..a286dada 100755 --- a/bin/Keys.py +++ b/bin/Keys.py @@ -17,7 +17,9 @@ def search_gpg(message): if '-----BEGIN PGP MESSAGE-----' in content: publisher.warning('{} has a PGP enc message'.format(paste.p_name)) #Send to duplicate - p.populate_set_out(message) + p.populate_set_out(message, 'Duplicate') + #send to Browse_warning_paste + p.populate_set_out('keys;{}'.format(message), 'BrowseWarningPaste') if __name__ == '__main__': @@ -49,4 +51,3 @@ if __name__ == '__main__': search_gpg(message) # (Optional) Send that thing to the next queue - #p.populate_set_out(something_has_been_done) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index d6706e1e..7f1220c9 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -67,6 +67,7 @@ function launching_lvldb { db1_y='2013' db2_y='2014' db3_y='2016' + db4_y='3016' nb_db=13 screen -dmS "LevelDB" @@ -78,6 +79,10 @@ function launching_lvldb { screen -S "LevelDB" -X screen -t "2014" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2014/ -P '$db2_y' -M '$nb_db'; read x' sleep 0.1 screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db3_y' -M '$nb_db'; read x' + + # For Curve + sleep 0.1 + screen -S "LevelDB" -X screen -t "3016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'3016/ -P '$db4_y' -M '$nb_db'; read x' } function launching_logs { @@ -140,6 +145,12 @@ function launching_scripts { screen -S "Script" -X screen -t "Cve" bash -c './Cve.py; read x' sleep 0.1 screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "ModuleStats" bash -c './ModuleStats.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "Browse_warning_paste" bash -c './Browse_warning_paste.py; read x' } #If no params, display the help diff --git a/bin/Mail.py b/bin/Mail.py index 964deb19..2b3ed5fc 100755 --- a/bin/Mail.py +++ b/bin/Mail.py @@ -61,9 +61,16 @@ if __name__ == "__main__": if MX_values[0] > is_critical: publisher.warning(to_print) #Send to duplicate - p.populate_set_out(filename) + p.populate_set_out(filename, 'Duplicate') + else: publisher.info(to_print) + #Send to ModuleStats + for mail in MX_values[1]: + print 'mail;{};{};{}'.format(1, mail, PST.p_date) + p.populate_set_out('mail;{};{};{}'.format(1, mail, PST.p_date), 'ModuleStats') + p.populate_set_out('mail;{}'.format(filename), 'BrowseWarningPaste') + prec_filename = filename else: diff --git a/bin/ModuleStats.py b/bin/ModuleStats.py new file mode 100755 index 00000000..32c631ed --- /dev/null +++ b/bin/ModuleStats.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python2 +# -*-coding:UTF-8 -* +""" + Template for new modules +""" + +import time +import datetime +import redis +import os +from packages import lib_words +from packages.Date import Date +from pubsublogger import publisher +from Helper import Process +from packages import Paste + +# Config Var +max_set_cardinality = 7 + +def get_date_range(num_day): + curr_date = datetime.date.today() + date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2)) + date_list = [] + + for i in range(0, num_day+1): + date_list.append(date.substract_day(i)) + return date_list + + +def compute_most_posted(server, message): + module, num, keyword, paste_date = message.split(';') + + redis_progression_name_set = 'top_'+ module +'_set' + + # Add/Update in Redis + prev_score = server.hget(paste_date, module+'-'+keyword) + if prev_score is not None: + ok = server.hset(paste_date, module+'-'+keyword, int(prev_score) + int(num)) + else: + ok = server.hset(paste_date, module+'-'+keyword, int(num)) + + # Compute Most Posted + date = get_date_range(0)[0] + # check if this keyword is eligible for progression + keyword_total_sum = 0 + + curr_value = server.hget(date, module+'-'+keyword) + keyword_total_sum += int(curr_value) if curr_value is not None else 0 + + if keyword in server.smembers(redis_progression_name_set): # if it is already in the set + return + + if (server.scard(redis_progression_name_set) < max_set_cardinality): + server.sadd(redis_progression_name_set, keyword) + + else: #not in the set + #Check value for all members + member_set = [] + for keyw in server.smembers(redis_progression_name_set): + keyw_value = server.hget(paste_date, module+'-'+keyw) + if keyw_value is not None: + member_set.append((keyw, int(keyw_value))) + else: #No data for this set for today + member_set.append((keyw, int(0))) + member_set.sort(key=lambda tup: tup[1]) + if len(member_set) > 0: + if member_set[0][1] < keyword_total_sum: + #remove min from set and add the new one + print module + ': adding ' +keyword+ '(' +str(keyword_total_sum)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')' + server.srem(redis_progression_name_set, member_set[0][0]) + server.sadd(redis_progression_name_set, keyword) + + +def compute_provider_info(server, path): + + redis_avg_size_name_set = 'top_size_set' + redis_providers_name_set = 'providers_set' + paste = Paste.Paste(path) + + paste_size = paste._get_p_size() + paste_provider = paste.p_source + paste_date = paste._get_p_date() + new_avg = paste_size + + # Add/Update in Redis + prev_num_paste = server.hget(paste_provider+'_num', paste_date) + if prev_num_paste is not None: + ok = server.hset(paste_provider+'_num', paste_date, int(prev_num_paste)+1) + prev_sum_size = server.hget(paste_provider+'_size', paste_date) + + if prev_sum_size is not None: + ok = server.hset(paste_provider+'_size', paste_date, float(prev_sum_size)+paste_size) + new_avg = (float(prev_sum_size)+paste_size) / (int(prev_num_paste)+1) + else: + ok = server.hset(paste_provider+'_size', paste_date, paste_size) + + else: + ok = server.hset(paste_provider+'_num', paste_date, 1) + prev_num_paste = 0 + + # + # Compute Most Posted + # + + # Size + if paste_provider not in server.smembers(redis_avg_size_name_set): # if it is already in the set + if (server.scard(redis_avg_size_name_set) < max_set_cardinality): + server.sadd(redis_avg_size_name_set, paste_provider) + + else: #set full capacity + #Check value for all members + member_set = [] + for provider in server.smembers(redis_avg_size_name_set): + curr_avg = 0.0 + curr_size = server.hget(provider+'_size', paste_date) + curr_num = server.hget(provider+'_num', paste_date) + if (curr_size is not None) and (curr_num is not None): + curr_avg = float(curr_size) / float(curr_num) + member_set.append((provider, curr_avg)) + member_set.sort(key=lambda tup: tup[1]) + if member_set[0][1] < new_avg: + #remove min from set and add the new one + print 'Size - adding ' +paste_provider+ '(' +str(new_avg)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')' + server.srem(redis_avg_size_name_set, member_set[0][0]) + server.sadd(redis_avg_size_name_set, paste_provider) + + # Num + if paste_provider not in server.smembers(redis_providers_name_set): # if it is already in the set + if (server.scard(redis_providers_name_set) < max_set_cardinality): + server.sadd(redis_providers_name_set, paste_provider) + + else: #set full capacity + #Check value for all members + member_set = [] + for provider in server.smembers(redis_providers_name_set): + curr_num = 0 + curr_num = server.hget(provider+'_num', paste_date) + if curr_num is not None: + member_set.append((provider, int(curr_num))) + member_set.sort(key=lambda tup: tup[1]) + if len(member_set) > 0: + if member_set[0][1] < int(prev_num_paste)+1: + #remove min from set and add the new one + print 'Num - adding ' +paste_provider+ '(' +str(int(prev_num_paste)+1)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')' + server.srem(redis_providers_name_set, member_set[0][0]) + server.sadd(redis_providers_name_set, paste_provider) + +if __name__ == '__main__': + # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh) + # Port of the redis instance used by pubsublogger + publisher.port = 6380 + # Script is the default channel used for the modules. + publisher.channel = 'Script' + + # Section name in bin/packages/modules.cfg + config_section = 'ModuleStats' + + # Setup the I/O queues + p = Process(config_section) + + # Sent to the logging a description of the module + publisher.info("Makes statistics about valid URL") + + # REDIS # + r_serv_trend = redis.StrictRedis( + host=p.config.get("Redis_Level_DB_Trending", "host"), + port=p.config.get("Redis_Level_DB_Trending", "port"), + db=p.config.get("Redis_Level_DB_Trending", "db")) + + # Endless loop getting messages from the input queue + while True: + # Get one message from the input queue + message = p.get_from_set() + + if message is None: + publisher.debug("{} queue is empty, waiting".format(config_section)) + print 'sleeping' + time.sleep(20) + continue + + else: + # Do something with the message from the queue + if len(message.split(';')) > 1: + compute_most_posted(r_serv_trend, message) + else: + compute_provider_info(r_serv_trend, message) diff --git a/bin/Phone.py b/bin/Phone.py index b53b079c..b25dae41 100755 --- a/bin/Phone.py +++ b/bin/Phone.py @@ -23,8 +23,10 @@ def search_phone(message): if len(results) > 4: print results publisher.warning('{} contains PID (phone numbers)'.format(paste.p_name)) + #send to Browse_warning_paste + p.populate_set_out('phone;{}'.format(message), 'BrowseWarningPaste') #Send to duplicate - p.populate_set_out(message) + p.populate_set_out(message, 'Duplicate') if __name__ == '__main__': # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh) diff --git a/bin/SQLInjectionDetection.py b/bin/SQLInjectionDetection.py new file mode 100755 index 00000000..b2d002fe --- /dev/null +++ b/bin/SQLInjectionDetection.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python2 +# -*-coding:UTF-8 -* +""" + Sql Injection module +""" + +import time +import string +import urllib2 +import re +from pubsublogger import publisher +from Helper import Process +from packages import Paste +from pyfaup.faup import Faup + +# Config Var + +regex_injection = [] +word_injection = [] +word_injection_suspect = [] + +# Classic atome injection +regex_injection1 = "([[AND |OR ]+[\'|\"]?[0-9a-zA-Z]+[\'|\"]?=[\'|\"]?[0-9a-zA-Z]+[\'|\"]?])" +regex_injection.append(regex_injection1) + +# Time-based attack +regex_injection2 = ["SLEEP\([0-9]+", "BENCHMARK\([0-9]+", "WAIT FOR DELAY ", "WAITFOR DELAY"] +regex_injection2 = re.compile('|'.join(regex_injection2)) +regex_injection.append(regex_injection2) + +# Interesting keyword +word_injection1 = [" IF ", " ELSE ", " CASE ", " WHEN ", " END ", " UNION ", "SELECT ", " FROM ", " ORDER BY ", " WHERE ", " DELETE ", " DROP ", " UPDATE ", " EXEC "] +word_injection.append(word_injection1) + +# Database special keywords +word_injection2 = ["@@version", "POW(", "BITAND(", "SQUARE("] +word_injection.append(word_injection2) + +# Html keywords +word_injection3 = [" + + + + + + + + +
+ +
+
+ {% include 'trending_graphs/Moduletrending.html' %} +
+ + + + + + + +
+ + + + diff --git a/var/www/templates/Trending.html b/var/www/templates/Trending.html index f99a88f9..92c31e49 100644 --- a/var/www/templates/Trending.html +++ b/var/www/templates/Trending.html @@ -5,6 +5,10 @@ + + + + Analysis Information Leak framework Dashboard @@ -15,7 +19,10 @@ - + + + @@ -29,6 +36,8 @@ @@ -58,26 +67,34 @@
+ + -
+ + +
{% include 'trending_graphs/Tldstrending.html' %}
{% include 'trending_graphs/Domainstrending.html' %}
-
- {% include 'trending_graphs/Wordstrending.html' %} -
{% include 'trending_graphs/Protocolstrending.html' %}
+
+ {% include 'trending_graphs/Wordstrending.html' %} +
@@ -86,6 +103,25 @@ + + + + + + + + + + + +
+ + + + + + +
+
+
+

Browse important pastes

+
+ +
+ +
+ + + +
+ + +
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+ + + + + + + + + +
+ + + diff --git a/var/www/templates/important_paste_by_module.html b/var/www/templates/important_paste_by_module.html new file mode 100644 index 00000000..164ed800 --- /dev/null +++ b/var/www/templates/important_paste_by_module.html @@ -0,0 +1,150 @@ + + + + + + + + + + + + {% set i = 0 %} + {% for path in all_path %} + + + + + + + + {% set i = i + 1 %} + {% endfor %} + +
#PathDate# of linesAction
{{ i + 1 }}{{ path }}{{ paste_date[i] }}{{ paste_linenum[i] }}

+
+
+ + + + + + diff --git a/var/www/templates/index.html b/var/www/templates/index.html index f4559450..b6426ea7 100644 --- a/var/www/templates/index.html +++ b/var/www/templates/index.html @@ -18,6 +18,7 @@