diff --git a/bin/BankAccount.py b/bin/BankAccount.py index 16a8a11f..d3cb4c16 100755 --- a/bin/BankAccount.py +++ b/bin/BankAccount.py @@ -5,7 +5,7 @@ The BankAccount Module ====================== -It apply IBAN regexes on paste content and warn if above a threshold. +It apply IBAN regexes on item content and warn if above a threshold. """ @@ -17,7 +17,7 @@ import re import string from itertools import chain -from packages import Paste +from packages import Item from pubsublogger import publisher from Helper import Process @@ -49,7 +49,7 @@ def is_valid_iban(iban): return True return False -def check_all_iban(l_iban, paste, filename): +def check_all_iban(l_iban, obj_id): nb_valid_iban = 0 for iban in l_iban: iban = iban[0]+iban[1]+iban[2] @@ -65,14 +65,14 @@ def check_all_iban(l_iban, paste, filename): server_statistics.hincrby('iban_by_country:'+date, iban[0:2], 1) if(nb_valid_iban > 0): - to_print = 'Iban;{};{};{};'.format(paste.p_source, paste.p_date, paste.p_name) + to_print = 'Iban;{};{};{};'.format(Item.get_source(obj_id), Item.get_item_date(obj_id), Item.get_basename(obj_id)) publisher.warning('{}Checked found {} IBAN;{}'.format( - to_print, nb_valid_iban, paste.p_rel_path)) - msg = 'infoleak:automatic-detection="iban";{}'.format(filename) + to_print, nb_valid_iban, obj_id)) + msg = 'infoleak:automatic-detection="iban";{}'.format(obj_id) p.populate_set_out(msg, 'Tags') #Send to duplicate - p.populate_set_out(filename, 'Duplicate') + p.populate_set_out(obj_id, 'Duplicate') if __name__ == "__main__": publisher.port = 6380 @@ -103,21 +103,21 @@ if __name__ == "__main__": if message is not None: - filename = message - paste = Paste.Paste(filename) - content = paste.get_p_content() + obj_id = Item.get_item_id(message) + + content = Item.get_item_content(obj_id) signal.alarm(max_execution_time) try: l_iban = iban_regex.findall(content) except TimeoutException: - print ("{0} processing timeout".format(paste.p_rel_path)) + print ("{0} processing timeout".format(obj_id)) continue else: signal.alarm(0) if(len(l_iban) > 0): - check_all_iban(l_iban, paste, filename) + check_all_iban(l_iban, obj_id) else: publisher.debug("Script BankAccount is Idling 10s") diff --git a/bin/Decoder.py b/bin/Decoder.py index c841876f..84e32178 100755 --- a/bin/Decoder.py +++ b/bin/Decoder.py @@ -17,7 +17,6 @@ import datetime from pubsublogger import publisher from Helper import Process -from packages import Paste from packages import Item from lib import Decoded @@ -56,14 +55,14 @@ def decode_string(content, item_id, item_date, encoded_list, decoder_name, encod Decoded.save_item_relationship(sha1_string, item_id) Decoded.create_decoder_matadata(sha1_string, item_id, decoder_name) - #remove encoded from paste content + #remove encoded from item content content = content.replace(encoded, '', 1) if(find): - set_out_paste(decoder_name, item_id) + set_out_item(decoder_name, message) return content -def set_out_paste(decoder_name, message): +def set_out_item(decoder_name, message): publisher.warning(decoder_name+' decoded') #Send to duplicate p.populate_set_out(message, 'Duplicate') @@ -130,12 +129,11 @@ if __name__ == '__main__': time.sleep(1) continue - filename = message - paste = Paste.Paste(filename) + obj_id = Item.get_item_id(message) # Do something with the message from the queue - content = paste.get_p_content() - date = str(paste._get_p_date()) + content = Item.get_item_content(obj_id) + date = Item.get_item_date(obj_id) for decoder in decoder_order: # add threshold and size limit @@ -146,7 +144,7 @@ if __name__ == '__main__': except TimeoutException: encoded_list = [] p.incr_module_timeout_statistic() # add encoder type - print ("{0} processing timeout".format(paste.p_rel_path)) + print ("{0} processing timeout".format(obj_id)) continue else: signal.alarm(0) diff --git a/bin/Global.py b/bin/Global.py index 00207c63..3bf36215 100755 --- a/bin/Global.py +++ b/bin/Global.py @@ -21,14 +21,35 @@ Requirements """ import base64 +import hashlib +import io +import gzip import os +import sys import time import uuid + +import datetime +import redis + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) +import ConfigLoader + from pubsublogger import publisher from Helper import Process -import magic +config_loader = ConfigLoader.ConfigLoader() +r_stats = config_loader.get_redis_conn("ARDB_Statistics") +config_loader = None + +def gunzip_bytes_obj(bytes_obj): + in_ = io.BytesIO() + in_.write(bytes_obj) + in_.seek(0) + with gzip.GzipFile(fileobj=in_, mode='rb') as fo: + gunzipped_bytes_obj = fo.read() + return gunzipped_bytes_obj def rreplace(s, old, new, occurrence): li = s.rsplit(old, occurrence) @@ -45,8 +66,10 @@ if __name__ == '__main__': p = Process(config_section) + # get and sanityze PASTE DIRECTORY PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], p.config.get("Directories", "pastes")) PASTES_FOLDERS = PASTES_FOLDER + '/' + PASTES_FOLDERS = os.path.join(os.path.realpath(PASTES_FOLDERS), '') # LOGGING # publisher.info("Feed Script started to receive & publish.") @@ -65,9 +88,9 @@ if __name__ == '__main__': publisher.debug("Empty Paste: {0} not processed".format(message)) continue else: - print("Empty Queues: Waiting...") + #print("Empty Queues: Waiting...") if int(time.time() - time_1) > 30: - to_print = 'Global; ; ; ;glob Processed {0} paste(s)'.format(processed_paste) + to_print = 'Global; ; ; ;glob Processed {0} paste(s) in {1} s'.format(processed_paste, time.time() - time_1) print(to_print) #publisher.info(to_print) time_1 = time.time() @@ -75,6 +98,10 @@ if __name__ == '__main__': time.sleep(1) continue + # remove PASTES_FOLDER from item path (crawled item + submited) + if PASTES_FOLDERS in paste: + paste = paste.replace(PASTES_FOLDERS, '', 1) + file_name_paste = paste.split('/')[-1] if len(file_name_paste)>255: new_file_name_paste = '{}{}.gz'.format(file_name_paste[:215], str(uuid.uuid4())) @@ -82,33 +109,73 @@ if __name__ == '__main__': # Creating the full filepath filename = os.path.join(PASTES_FOLDER, paste) + filename = os.path.realpath(filename) - dirname = os.path.dirname(filename) - if not os.path.exists(dirname): - os.makedirs(dirname) + # incorrect filename + if not os.path.commonprefix([filename, PASTES_FOLDER]) == PASTES_FOLDER: + print('Path traversal detected {}'.format(filename)) + publisher.warning('Global; Path traversal detected') + else: - decoded = base64.standard_b64decode(gzip64encoded) + # decode compressed base64 + decoded = base64.standard_b64decode(gzip64encoded) - with open(filename, 'wb') as f: - f.write(decoded) - '''try: - decoded2 = gunzip_bytes_obj(decoded) - except: - decoded2 ='' + # check if file exist + if os.path.isfile(filename): + print('File already exist {}'.format(filename)) + publisher.warning('Global; File already exist') - type = magic.from_buffer(decoded2, mime=True) + try: + with gzip.open(filename, 'rb') as f: + curr_file_content = f.read() + except EOFError: + publisher.warning('Global; Incomplete file: {}'.format(filename)) + # save daily stats + r_stats.zincrby('module:Global:incomplete_file', datetime.datetime.now().strftime('%Y%m%d'), 1) + # discard item + continue + except OSError: + publisher.warning('Global; Not a gzipped file: {}'.format(filename)) + # save daily stats + r_stats.zincrby('module:Global:invalid_file', datetime.datetime.now().strftime('%Y%m%d'), 1) + # discard item + continue - if type!= 'text/x-c++' and type!= 'text/html' and type!= 'text/x-c' and type!= 'text/x-python' and type!= 'text/x-php' and type!= 'application/xml' and type!= 'text/x-shellscript' and type!= 'text/plain' and type!= 'text/x-diff' and type!= 'text/x-ruby': + curr_file_md5 = hashlib.md5(curr_file_content).hexdigest() - print('-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------') - print(filename) - print(type) - print('-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------') - ''' + new_file_content = gunzip_bytes_obj(decoded) + new_file_md5 = hashlib.md5(new_file_content).hexdigest() - # remove PASTES_FOLDER from item path (crawled item + submited) - if PASTES_FOLDERS in paste: - paste = paste.replace(PASTES_FOLDERS, '', 1) + if new_file_md5 != curr_file_md5: - p.populate_set_out(paste) - processed_paste+=1 + if filename.endswith('.gz'): + filename = '{}_{}.gz'.format(filename[:-3], new_file_md5) + else: + filename = '{}_{}'.format(filename, new_file_md5) + + # continue if new file already exist + if os.path.isfile(filename): + print('ignore duplicated file') + continue + + print('new file: {}'.format(filename)) + # ignore duplicate + else: + print('ignore duplicated file') + continue + + # create subdir + dirname = os.path.dirname(filename) + if not os.path.exists(dirname): + os.makedirs(dirname) + + with open(filename, 'wb') as f: + f.write(decoded) + + paste = filename + # remove PASTES_FOLDER from + if PASTES_FOLDERS in paste: + paste = paste.replace(PASTES_FOLDERS, '', 1) + + p.populate_set_out(paste) + processed_paste+=1 diff --git a/bin/IPAddress.py b/bin/IPAddress.py index f03ee8b3..e2ed59cf 100755 --- a/bin/IPAddress.py +++ b/bin/IPAddress.py @@ -6,7 +6,7 @@ The IP Module This module is consuming the global channel. -It first performs a regex to find IP addresses and then matches those IPs to +It first performs a regex to find IP addresses and then matches those IPs to some configured ip ranges. The list of IP ranges are expected to be in CIDR format (e.g. 192.168.0.0/16) @@ -16,6 +16,7 @@ and should be defined in the config.cfg file, under the [IP] section import time import re +import sys from pubsublogger import publisher from packages import Paste from Helper import Process @@ -31,8 +32,9 @@ def search_ip(message): results = reg_ip.findall(content) matching_ips = [] - for res in results: - address = IPv4Address(res) + for ip in results: + ip = '.'.join([str(int(x)) for x in ip.split('.')]) + address = IPv4Address(ip) for network in ip_networks: if address in network: matching_ips.append(address) @@ -60,8 +62,12 @@ if __name__ == '__main__': p = Process(config_section) ip_networks = [] - for network in p.config.get("IP", "networks").split(","): - ip_networks.append(IPv4Network(network)) + try: + for network in p.config.get("IP", "networks").split(","): + ip_networks.append(IPv4Network(network)) + except: + print('Please provide a list of valid IP addresses') + sys.exit(0) # Sent to the logging a description of the module @@ -78,4 +84,3 @@ if __name__ == '__main__': # Do something with the message from the queue search_ip(message) - diff --git a/bin/NotificationHelper.py b/bin/NotificationHelper.py index 02568a1e..b9c90104 100755 --- a/bin/NotificationHelper.py +++ b/bin/NotificationHelper.py @@ -26,6 +26,7 @@ publisher.channel = "Script" def sendEmailNotification(recipient, alert_name, content): sender = config_loader.get_config_str("Notifications", "sender") + sender_user = config_loader.get_config_str("Notifications", "sender_user") sender_host = config_loader.get_config_str("Notifications", "sender_host") sender_port = config_loader.get_config_int("Notifications", "sender_port") sender_pw = config_loader.get_config_str("Notifications", "sender_pw") @@ -49,7 +50,10 @@ def sendEmailNotification(recipient, alert_name, content): smtp_server = smtplib.SMTP_SSL(sender_host, sender_port) smtp_server.ehlo() - smtp_server.login(sender, sender_pw) + if sender_user is not None: + smtp_server.login(sender_user, sender_pw) + else: + smtp_server.login(sender, sender_pw) else: smtp_server = smtplib.SMTP(sender_host, sender_port) diff --git a/bin/SentimentAnalysis.py b/bin/SentimentAnalysis.py index fc9a2f4c..a90a3a09 100755 --- a/bin/SentimentAnalysis.py +++ b/bin/SentimentAnalysis.py @@ -30,7 +30,7 @@ sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader from nltk.sentiment.vader import SentimentIntensityAnalyzer -from nltk import tokenize +from nltk import tokenize, download # Config Variables accepted_Mime_type = ['text/plain'] @@ -62,7 +62,12 @@ def Analyse(message, server): combined_datetime = datetime.datetime.combine(the_date, the_time) timestamp = calendar.timegm(combined_datetime.timetuple()) - sentences = tokenize.sent_tokenize(p_content) + try: + sentences = tokenize.sent_tokenize(p_content) + except: + # use the NLTK Downloader to obtain the resource + download('punkt') + sentences = tokenize.sent_tokenize(p_content) if len(sentences) > 0: avg_score = {'neg': 0.0, 'neu': 0.0, 'pos': 0.0, 'compoundPos': 0.0, 'compoundNeg': 0.0} diff --git a/bin/WebStats.py b/bin/WebStats.py index 7eecb0d2..10aba917 100755 --- a/bin/WebStats.py +++ b/bin/WebStats.py @@ -29,7 +29,10 @@ num_day_to_look = 5 # the detection of the progression start num_day_to_lo def analyse(server, field_name, date, url_parsed): field = url_parsed[field_name] if field is not None: - field = field.decode('utf8') + try: # faup version + field = field.decode() + except: + pass server.hincrby(field, date, 1) if field_name == "domain": #save domain in a set for the monthly plot domain_set_name = "domain_set_" + date[0:6] diff --git a/bin/packages/Item.py b/bin/packages/Item.py index 530e6408..1046c154 100755 --- a/bin/packages/Item.py +++ b/bin/packages/Item.py @@ -37,6 +37,9 @@ def exist_item(item_id): else: return False +def get_basename(item_id): + return os.path.basename(item_id) + def get_item_id(full_path): return full_path.replace(PASTES_FOLDER, '', 1) diff --git a/bin/packages/Tag.py b/bin/packages/Tag.py index a062da5e..0c6a695b 100755 --- a/bin/packages/Tag.py +++ b/bin/packages/Tag.py @@ -394,6 +394,11 @@ def add_tag(object_type, tag, object_id, obj_date=None): # new tag if not is_obj_tagged(object_id, tag): # # TODO: # FIXME: sanityze object_type + if obj_date: + try: + obj_date = int(obj_date) + except: + obj_date = None if not obj_date: obj_date = get_obj_date(object_type, object_id) add_global_tag(tag, object_type=object_type) @@ -455,7 +460,7 @@ def delete_obj_tags(object_id, object_type, tags=[]): return res def sanitise_tags_date_range(l_tags, date_from=None, date_to=None): - if date_from or date_to is None: + if date_from is None or date_to is None: date_from = get_tags_min_last_seen(l_tags, r_int=False) date_to = date_from return Date.sanitise_date_range(date_from, date_to) diff --git a/configs/core.cfg.sample b/configs/core.cfg.sample index 9cea213f..876bd5ff 100644 --- a/configs/core.cfg.sample +++ b/configs/core.cfg.sample @@ -28,6 +28,9 @@ sender = sender@example.com sender_host = smtp.example.com sender_port = 1337 sender_pw = None +# Only needed when the credentials for email server needs a username instead of an email address +#sender_user = sender +sender_user = # optional for using with authenticated SMTP over SSL # sender_pw = securepassword diff --git a/var/www/blueprints/crawler_splash.py b/var/www/blueprints/crawler_splash.py index ee2e68ba..7acecfbe 100644 --- a/var/www/blueprints/crawler_splash.py +++ b/var/www/blueprints/crawler_splash.py @@ -154,5 +154,5 @@ def domains_explorer_web(): except: page = 1 - dict_data = Domain.domains_up_by_page('regular', page=page) + dict_data = Domain.get_domains_up_by_filers('regular', page=page, date_from=date_from, date_to=date_to) return render_template("domain_explorer.html", dict_data=dict_data, bootstrap_label=bootstrap_label, domain_type='regular') diff --git a/var/www/templates/crawler/crawler_splash/domain_explorer.html b/var/www/templates/crawler/crawler_splash/domain_explorer.html index 007da8f0..16595014 100644 --- a/var/www/templates/crawler/crawler_splash/domain_explorer.html +++ b/var/www/templates/crawler/crawler_splash/domain_explorer.html @@ -83,6 +83,9 @@ {%else%} {% set target_url=url_for('crawler_splash.domains_explorer_web') + "?domain_type=regular" %} {%endif%} + {%if 'date_from' in dict_data %} + {% set target_url = target_url + '&date_from=' + dict_data['date_from'] + '&date_to=' + dict_data['date_to'] %} + {%endif%} {% include 'pagination.html' %} {% endwith %} {%endif%} diff --git a/var/www/templates/domains/domain_explorer.html b/var/www/templates/domains/domain_explorer.html deleted file mode 100644 index ec6211c7..00000000 --- a/var/www/templates/domains/domain_explorer.html +++ /dev/null @@ -1,239 +0,0 @@ - - - - Show Domain - AIL - - - - - - - - - - - - - - - {% include 'nav_bar.html' %} - -
-
- - {% include 'crawler/menu_sidebar.html' %} - -
- -
-
-
-
- -
-
- -
-
- -
-
-
-
- - {% for dict_domain in dict_data['list_elem'] %} - - {% if loop.index0 % 4 == 0 %} -
- {% endif %} - -
-
- - -
-
-
- - {{dict_domain["id"]}} - -
-

- - First seen: {{dict_domain["first_seen"]}}
- Last_seen: {{dict_domain["first_seen"]}}
- Ports: {{dict_domain["ports"]}} - -

- Status: - {% if dict_domain["status"] %} - - - UP - - {% else %} - - - DOWN - - {% endif %} -
- {% for tag in dict_domain['tags'] %} - - {{ tag }} - - {% endfor %} -
-
-
- - - {% if loop.index0 % 4 == 3 %} -
- {% endif %} - - {% endfor %} - - {% if dict_data['list_elem']|length % 4 != 0 %} -
- {% endif %} - -
-
- - {%if 'list_elem' in dict_data%} - {% with page=dict_data['page'], nb_page_max=dict_data['nb_pages'], nb_first_elem=dict_data['nb_first_elem'], nb_last_elem=dict_data['nb_last_elem'], nb_all_elem=dict_data['nb_all_elem'] %} - {% set object_name="domain" %} - {%if domain_type=='onion'%} - {% set target_url=url_for('crawler_splash.domains_explorer_onion') + "?domain_type=onion" %} - {%else%} - {% set target_url=url_for('crawler_splash.domains_explorer_web') + "?domain_type=regular" %} - {%endif%} - {% include 'pagination.html' %} - {% endwith %} - {%endif%} - - - - - -
-
- - - - - - - - - - -