From 126ecb2e39e88665163b842674093e9836ae0570 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 16 Mar 2023 16:49:53 +0100 Subject: [PATCH] fix: [core] fix merge --- bin/LAUNCH.sh | 4 +- .../ail_json_importer/Ail_bgp_monitor.py | 57 ----------------- bin/importer/feeders/BgpMonitor.py | 35 +++++++++++ bin/lib/crawlers.py | 17 ------ bin/packages/Term.py | 2 +- bin/trackers/Tracker_Typo_Squatting.py | 61 ++++++------------- update/v4.2.1/Update.py | 16 ++--- var/www/modules/hunter/Flask_hunter.py | 19 +++--- 8 files changed, 72 insertions(+), 139 deletions(-) delete mode 100755 bin/import/ail_json_importer/Ail_bgp_monitor.py create mode 100755 bin/importer/feeders/BgpMonitor.py diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 1fa0c3f9..efbb7f22 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -282,10 +282,10 @@ function launching_scripts { ################################## # TRACKERS MODULES # ################################## - screen -S "Script_AIL" -X screen -t "Tracker_Typo_Squatting" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Typo_Squatting.py; read x" - sleep 0.1 screen -S "Script_AIL" -X screen -t "Tracker_Term" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Term.py; read x" sleep 0.1 + screen -S "Script_AIL" -X screen -t "Tracker_Typo_Squatting" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Typo_Squatting.py; read x" + sleep 0.1 screen -S "Script_AIL" -X screen -t "Tracker_Regex" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Regex.py; read x" sleep 0.1 screen -S "Script_AIL" -X screen -t "Tracker_Yara" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Yara.py; read x" diff --git a/bin/import/ail_json_importer/Ail_bgp_monitor.py b/bin/import/ail_json_importer/Ail_bgp_monitor.py deleted file mode 100755 index e76760d5..00000000 --- a/bin/import/ail_json_importer/Ail_bgp_monitor.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* -""" -The JSON Receiver Module -================ - -Recieve Json Items (example: Twitter feeder) - -""" -import os -import json -import sys -import datetime -import uuid - -from packages import Tag - -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) -import item_basic - -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'import', 'ail_json_importer')) -from Default_json import Default_json - -class Ail_bgp_monitor(Default_json): - """urlextract Feeder functions""" - - def __init__(self, name, json_item): - super().__init__(name, json_item) - - def get_feeder_name(self): - return 'bgp_monitor' - - # # define item id - # def get_item_id(self): - # # use twitter timestamp ? - # item_date = datetime.date.today().strftime("%Y/%m/%d") - # item_id = str(self.json_item['meta']['twitter:url-extracted']) - # item_id = item_id.split('//') - # if len(item_id) > 1: - # item_id = ''.join(item_id[1:]) - # else: - # item_id = item_id[0] - # item_id = item_id.replace('/', '_') - # if len(item_id) > 215: - # item_id = '{}{}.gz'.format(item_id[:215], str(uuid.uuid4())) - # else: - # item_id = '{}{}.gz'.format(item_id, str(uuid.uuid4())) - # return os.path.join('urlextract', item_date, item_id) - - def process_json_meta(self, process, item_id): - ''' - Process JSON meta filed. - ''' - json_meta = self.get_json_meta() - - tag = 'infoleak:automatic-detection=bgp_monitor' - Tag.add_tag('item', tag, item_id) diff --git a/bin/importer/feeders/BgpMonitor.py b/bin/importer/feeders/BgpMonitor.py new file mode 100755 index 00000000..dc926bfd --- /dev/null +++ b/bin/importer/feeders/BgpMonitor.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* +""" +The Bgp Monitor Feeder Importer Module +================ + +Process Bgp Monitor JSON + +""" +import os +import sys + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from importer.feeders.Default import DefaultFeeder +from lib.objects.Items import Item + + +class BgpMonitorFeeder(DefaultFeeder): + """BgpMonitorFeeder Feeder functions""" + + def __init__(self, json_data): + super().__init__(json_data) + self.name = 'bgp_monitor' + + def process_meta(self): + """ + Process JSON meta filed. + """ + # DIRTY FIX + tag = 'infoleak:automatic-detection=bgp_monitor' + item = Item(self.get_item_id()) + item.add_tag(tag) diff --git a/bin/lib/crawlers.py b/bin/lib/crawlers.py index 6ff8da1a..2ce75949 100755 --- a/bin/lib/crawlers.py +++ b/bin/lib/crawlers.py @@ -1146,7 +1146,6 @@ class CrawlerTask: def get_proxy(self): return r_crawler.hget(f'crawler:task:{self.uuid}', 'proxy') -<<<<<<< HEAD def get_parent(self): return r_crawler.hget(f'crawler:task:{self.uuid}', 'parent') @@ -1316,22 +1315,6 @@ def create_task(url, depth=1, har=True, screenshot=True, header=None, cookiejar= ## -- CRAWLER TASK -- ## -======= -def send_url_to_crawl_in_queue(crawler_mode, crawler_type, url): - print(f'{crawler_type}_crawler_priority_queue', f'{url};{crawler_mode}') - r_serv_onion.sadd(f'{crawler_type}_crawler_priority_queue', f'{url};{crawler_mode}') - # add auto crawled url for user UI - if crawler_mode == 'auto': - r_serv_onion.sadd(f'auto_crawler_url:{crawler_type}', url) - -def add_url_to_crawl_in_queue(url, crawler_mode='manual'): # crawler_type - #print(f'{crawler_type}_crawler_priority_queue', f'{url};{crawler_mode}') - r_serv_onion.sadd(f'{crawler_type}_crawler_priority_queue', f'{url};{crawler_mode}') - # CURRENTLY DISABLED - # # add auto crawled url for user UI - # if crawler_mode == 'auto': - # r_serv_onion.sadd(f'auto_crawler_url:{crawler_type}', url) ->>>>>>> master #### CRAWLER TASK API #### diff --git a/bin/packages/Term.py b/bin/packages/Term.py index abcc6f83..bc4fced0 100755 --- a/bin/packages/Term.py +++ b/bin/packages/Term.py @@ -102,7 +102,7 @@ def get_text_word_frequency(item_content, filtering=True): # # TODO: create all tracked words def get_tracked_words_list(): return list(r_serv_term.smembers('all:tracker:word')) - + def get_set_tracked_words_list(): set_list = r_serv_term.smembers('all:tracker:set') all_set_list = [] diff --git a/bin/trackers/Tracker_Typo_Squatting.py b/bin/trackers/Tracker_Typo_Squatting.py index 34116beb..c6f16780 100755 --- a/bin/trackers/Tracker_Typo_Squatting.py +++ b/bin/trackers/Tracker_Typo_Squatting.py @@ -13,18 +13,18 @@ The Tracker_Typo_Squatting Module import os import sys import time -import requests - sys.path.append(os.environ['AIL_BIN']) ################################## # Import Project packages ################################## from modules.abstract_module import AbstractModule -import NotificationHelper -from packages.Item import Item +from lib.objects.Items import Item from lib import Tracker +from exporter.MailExporter import MailExporterTracker +from exporter.WebHookExporter import WebHookExporterTracker + class Tracker_Typo_Squatting(AbstractModule): mail_body_template = "AIL Framework,\nNew occurrence for tracked Typo: {}\nitem id: {}\nurl: {}{}" @@ -37,12 +37,14 @@ class Tracker_Typo_Squatting(AbstractModule): self.pending_seconds = 5 - self.full_item_url = self.process.config.get("Notifications", "ail_domain") + "/object/item?id=" - - # loads typosquatting + # Refresh typo squatting self.typosquat_tracked_words_list = Tracker.get_typosquatting_tracked_words_list() self.last_refresh_typosquat = time.time() + # Exporter + self.exporters = {'mail': MailExporterTracker(), + 'webhook': WebHookExporterTracker()} + self.redis_logger.info(f"Module: {self.module_name} Launched") def compute(self, message): @@ -53,64 +55,39 @@ class Tracker_Typo_Squatting(AbstractModule): self.redis_logger.debug('Tracked typosquatting refreshed') print('Tracked typosquatting refreshed') - host, id = message.split() + host, item_id = message.split() # Cast message as Item for tracker in self.typosquat_tracked_words_list: if host in self.typosquat_tracked_words_list[tracker]: - item = Item(id) + item = Item(item_id) self.new_tracker_found(tracker, 'typosquatting', item) def new_tracker_found(self, tracker, tracker_type, item): item_id = item.get_id() - item_date = item.get_date() item_source = item.get_source() - #self.redis_logger.info(f'new tracked typo found: {tracker} in {item_id}') print(f'new tracked typosquatting found: {tracker} in {item_id}') self.redis_logger.warning(f'tracker typosquatting: {tracker} in {item_id}') - print(Tracker.get_tracker_uuid_list(tracker, tracker_type)) for tracker_uuid in Tracker.get_tracker_uuid_list(tracker, tracker_type): + tracker = Tracker.Tracker(tracker_uuid) + # Source Filtering - tracker_sources = Tracker.get_tracker_uuid_sources(tracker) + tracker_sources = tracker.get_sources() if tracker_sources and item_source not in tracker_sources: continue Tracker.add_tracked_item(tracker_uuid, item_id) - # Tags - tags_to_add = Tracker.get_tracker_tags(tracker_uuid) - for tag in tags_to_add: + for tag in tracker.get_tags(): msg = f'{tag};{item_id}' self.send_message_to_queue(msg, 'Tags') - mail_to_notify = Tracker.get_tracker_mails(tracker_uuid) - if mail_to_notify: - mail_subject = Tracker.get_email_subject(tracker_uuid) - mail_body = Tracker_Typo_Squatting.mail_body_template.format(tracker, item_id, self.full_item_url, item_id) - for mail in mail_to_notify: - NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body) - - # Webhook - webhook_to_post = Tracker.get_tracker_webhook(tracker_uuid) - if webhook_to_post: - json_request = {"trackerId": tracker_uuid, - "itemId": item_id, - "itemURL": self.full_item_url + item_id, - "tracker": tracker, - "itemSource": item_source, - "itemDate": item_date, - "tags": tags_to_add, - "emailNotification": f'{mail_to_notify}', - "trackerType": tracker_type - } - try: - response = requests.post(webhook_to_post, json=json_request) - if response.status_code >= 400: - self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: {response.reason}") - except: - self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: Something went wrong") + if tracker.mail_export(): + self.exporters['mail'].export(tracker, item) + if tracker.webhook_export(): + self.exporters['webhook'].export(tracker, item) if __name__ == '__main__': diff --git a/update/v4.2.1/Update.py b/update/v4.2.1/Update.py index 19085e78..f20a9184 100755 --- a/update/v4.2.1/Update.py +++ b/update/v4.2.1/Update.py @@ -2,17 +2,13 @@ # -*-coding:UTF-8 -* import os -import re import sys -import time -import redis -import datetime -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader - -sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin')) -from ail_updater import AIL_Updater +sys.path.append(os.environ['AIL_HOME']) +################################## +# Import Project packages +################################## +from update.bin.old_ail_updater import AIL_Updater class Updater(AIL_Updater): """default Updater.""" @@ -20,7 +16,7 @@ class Updater(AIL_Updater): def __init__(self, version): super(Updater, self).__init__(version) -if __name__ == '__main__': +if __name__ == '__main__': updater = Updater('v4.2.1') updater.run_update() diff --git a/var/www/modules/hunter/Flask_hunter.py b/var/www/modules/hunter/Flask_hunter.py index e4357053..921581e1 100644 --- a/var/www/modules/hunter/Flask_hunter.py +++ b/var/www/modules/hunter/Flask_hunter.py @@ -89,12 +89,12 @@ def tracked_menu_yara(): @login_required @login_read_only def tracked_menu_typosquatting(): - filter_type = 'typosquatting' + tracker_type = 'typosquatting' user_id = current_user.get_id() - user_term = Term.get_all_user_tracked_terms(user_id, filter_type=filter_type) - global_term = Term.get_all_global_tracked_terms(filter_type=filter_type) - return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type) - + user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type=tracker_type) + global_trackers = Tracker.get_global_trackers_metadata(tracker_type=tracker_type) + return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, + bootstrap_label=bootstrap_label, tracker_type=tracker_type) @hunter.route("/tracker/add", methods=['GET', 'POST']) @login_required @@ -241,10 +241,9 @@ def show_tracker(): else: typo_squatting = None - if date_from: res = Term.parse_get_tracker_term_item({'uuid': tracker_uuid, 'date_from': date_from, 'date_to': date_to}, user_id) - if res[1] !=200: + if res[1] != 200: return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] tracker_metadata['items'] = res[0]['items'] tracker_metadata['date_from'] = res[0]['date_from'] @@ -257,9 +256,9 @@ def show_tracker(): tracker_metadata['sources'] = sorted(tracker_metadata['sources']) return render_template("showTracker.html", tracker_metadata=tracker_metadata, - yara_rule_content=yara_rule_content, - typo_squatting=typo_squatting, - bootstrap_label=bootstrap_label) + yara_rule_content=yara_rule_content, + typo_squatting=typo_squatting, + bootstrap_label=bootstrap_label) @hunter.route("/tracker/update_tracker_description", methods=['POST']) @login_required