From 5f150489b689ada9fe7d9b8941887c58dd9bb908 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 30 Mar 2023 14:58:55 +0200 Subject: [PATCH] fix: [tracker] fix exporters --- bin/trackers/Tracker_Regex.py | 14 ++--- bin/trackers/Tracker_Term.py | 80 +++++++++++--------------- bin/trackers/Tracker_Typo_Squatting.py | 2 - bin/trackers/Tracker_Yara.py | 70 +++++++++------------- 4 files changed, 63 insertions(+), 103 deletions(-) diff --git a/bin/trackers/Tracker_Regex.py b/bin/trackers/Tracker_Regex.py index d213e42d..66e6514f 100755 --- a/bin/trackers/Tracker_Regex.py +++ b/bin/trackers/Tracker_Regex.py @@ -25,9 +25,6 @@ from exporter.MailExporter import MailExporterTracker from exporter.WebHookExporter import WebHookExporterTracker class Tracker_Regex(AbstractModule): - - mail_body_template = "AIL Framework,\nNew occurrence for tracked regex: {}\nitem id: {}\nurl: {}{}" - """ Tracker_Regex module for AIL framework """ @@ -66,18 +63,12 @@ class Tracker_Regex(AbstractModule): if matched: self.new_tracker_found(regex, 'regex', item) - # match = self.regex_finditer(self.dict_regex_tracked[regex], item_id, content) - # if match: - # self.new_tracker_found(regex, 'regex', item) - def new_tracker_found(self, tracker_name, tracker_type, item): uuid_list = Tracker.get_tracker_uuid_list(tracker_name, tracker_type) item_id = item.get_id() # date = item.get_date() item_source = item.get_source() - print(f'new tracked regex found: {tracker_name} in {item_id}') - self.redis_logger.warning(f'new tracked regex found: {tracker_name} in {item_id}') for tracker_uuid in uuid_list: tracker = Tracker.Tracker(tracker_uuid) @@ -87,7 +78,10 @@ class Tracker_Regex(AbstractModule): if tracker_sources and item_source not in tracker_sources: continue - Tracker.add_tracked_item(tracker_uuid, item_id) # TODO + print(f'new tracked regex found: {tracker_name} in {item_id}') + self.redis_logger.warning(f'new tracked regex found: {tracker_name} in {item_id}') + # TODO + Tracker.add_tracked_item(tracker_uuid, item_id) for tag in tracker.get_tags(): msg = f'{tag};{item_id}' diff --git a/bin/trackers/Tracker_Term.py b/bin/trackers/Tracker_Term.py index fa43fc1d..f3ee7ec7 100755 --- a/bin/trackers/Tracker_Term.py +++ b/bin/trackers/Tracker_Term.py @@ -13,7 +13,6 @@ import os import sys import time import signal -import requests sys.path.append(os.environ['AIL_BIN']) @@ -21,11 +20,13 @@ sys.path.append(os.environ['AIL_BIN']) # Import Project packages ################################## from modules.abstract_module import AbstractModule -import NotificationHelper from lib.objects.Items import Item from packages import Term from lib import Tracker +from exporter.MailExporter import MailExporterTracker +from exporter.WebHookExporter import WebHookExporterTracker + class TimeoutException(Exception): pass @@ -38,8 +39,6 @@ signal.signal(signal.SIGALRM, timeout_handler) class Tracker_Term(AbstractModule): - mail_body_template = "AIL Framework,\nNew occurrence for tracked term: {}\nitem id: {}\nurl: {}{}" - """ Tracker_Term module for AIL framework """ @@ -51,14 +50,16 @@ class Tracker_Term(AbstractModule): self.max_execution_time = self.process.config.getint('Tracker_Term', "max_execution_time") - self.full_item_url = self.process.config.get("Notifications", "ail_domain") + "/object/item?id=" - # loads tracked words self.list_tracked_words = Term.get_tracked_words_list() self.last_refresh_word = time.time() self.set_tracked_words_list = Term.get_set_tracked_words_list() self.last_refresh_set = time.time() + # Exporter + self.exporters = {'mail': MailExporterTracker(), + 'webhook': WebHookExporterTracker()} + self.redis_logger.info(f"Module: {self.module_name} Launched") def compute(self, item_id, item_content=None): @@ -77,7 +78,6 @@ class Tracker_Term(AbstractModule): # Cast message as Item item = Item(item_id) - item_date = item.get_date() if not item_content: item_content = item.get_content() @@ -115,52 +115,38 @@ class Tracker_Term(AbstractModule): if nb_uniq_word >= nb_words_threshold: self.new_term_found(word_set, 'set', item) - def new_term_found(self, term, term_type, item): - uuid_list = Term.get_term_uuid_list(term, term_type) + def new_term_found(self, tracker_name, tracker_type, item): + uuid_list = Tracker.get_tracker_uuid_list(tracker_name, tracker_type) item_id = item.get_id() - item_date = item.get_date() item_source = item.get_source() - self.redis_logger.warning(f'new tracked term found: {term} in {item_id}') - print(f'new tracked term found: {term} in {item_id}') - for term_uuid in uuid_list: - tracker_sources = Tracker.get_tracker_uuid_sources(term_uuid) - if not tracker_sources or item_source in tracker_sources: - Tracker.add_tracked_item(term_uuid, item_id) - tags_to_add = Term.get_term_tags(term_uuid) - for tag in tags_to_add: - msg = '{};{}'.format(tag, item_id) - self.send_message_to_queue(msg, 'Tags') + for tracker_uuid in uuid_list: + tracker = Tracker.Tracker(tracker_uuid) - mail_to_notify = Term.get_term_mails(term_uuid) - if mail_to_notify: - mail_subject = Tracker.get_email_subject(term_uuid) - mail_body = Tracker_Term.mail_body_template.format(term, item_id, self.full_item_url, item_id) - for mail in mail_to_notify: - self.redis_logger.debug(f'Send Mail {mail_subject}') - print(f'S print(item_content)end Mail {mail_subject}') - NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body) + # Source Filtering + tracker_sources = tracker.get_sources() + if tracker_sources and item_source not in tracker_sources: + continue - # Webhook - webhook_to_post = Term.get_term_webhook(term_uuid) - if webhook_to_post: - json_request = {"trackerId": term_uuid, - "itemId": item_id, - "itemURL": self.full_item_url + item_id, - "term": term, - "itemSource": item_source, - "itemDate": item_date, - "tags": tags_to_add, - "emailNotification": f'{mail_to_notify}', - "trackerType": term_type - } - try: - response = requests.post(webhook_to_post, json=json_request) - if response.status_code >= 400: - self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: {response.reason}") - except: - self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: Something went wrong") + print(f'new tracked term found: {tracker_name} in {item_id}') + self.redis_logger.warning(f'new tracked term found: {tracker_name} in {item_id}') + # TODO + Tracker.add_tracked_item(tracker_uuid, item_id) + + # Tags + for tag in tracker.get_tags(): + msg = f'{tag};{item_id}' + self.send_message_to_queue(msg, 'Tags') + + # Mail + if tracker.mail_export(): + # TODO add matches + custom subjects + self.exporters['mail'].export(tracker, item) + + # Webhook + if tracker.webhook_export(): + self.exporters['webhook'].export(tracker, item) if __name__ == '__main__': diff --git a/bin/trackers/Tracker_Typo_Squatting.py b/bin/trackers/Tracker_Typo_Squatting.py index c6f16780..898b137b 100755 --- a/bin/trackers/Tracker_Typo_Squatting.py +++ b/bin/trackers/Tracker_Typo_Squatting.py @@ -26,8 +26,6 @@ from exporter.MailExporter import MailExporterTracker from exporter.WebHookExporter import WebHookExporterTracker class Tracker_Typo_Squatting(AbstractModule): - mail_body_template = "AIL Framework,\nNew occurrence for tracked Typo: {}\nitem id: {}\nurl: {}{}" - """ Tracker_Typo_Squatting module for AIL framework """ diff --git a/bin/trackers/Tracker_Yara.py b/bin/trackers/Tracker_Yara.py index ec29aad8..b85deb55 100755 --- a/bin/trackers/Tracker_Yara.py +++ b/bin/trackers/Tracker_Yara.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 # -*-coding:UTF-8 -* -################################## -# The Tracker_Yara trackers module -################################## +""" +The Tracker_Yara trackers module +=================== + +""" ################################## # Import External packages @@ -11,23 +13,20 @@ import os import sys import time import yara -import requests sys.path.append(os.environ['AIL_BIN']) ################################## # Import Project packages ################################## from modules.abstract_module import AbstractModule -from packages import Term from lib.objects.Items import Item from lib import Tracker -import NotificationHelper # # TODO: refactor +from exporter.MailExporter import MailExporterTracker +from exporter.WebHookExporter import WebHookExporterTracker + class Tracker_Yara(AbstractModule): - - mail_body_template = "AIL Framework,\nNew YARA match: {}\nitem id: {}\nurl: {}{}" - """ Tracker_Yara module for AIL framework """ @@ -35,14 +34,16 @@ class Tracker_Yara(AbstractModule): super(Tracker_Yara, self).__init__() self.pending_seconds = 5 - self.full_item_url = self.process.config.get("Notifications", "ail_domain") + "/object/item?id=" - # Load Yara rules self.rules = Tracker.reload_yara_rules() self.last_refresh = time.time() self.item = None + # Exporter + self.exporters = {'mail': MailExporterTracker(), + 'webhook': WebHookExporterTracker()} + self.redis_logger.info(f"Module: {self.module_name} Launched") def compute(self, item_id, item_content=None): @@ -63,59 +64,40 @@ class Tracker_Yara(AbstractModule): if yara_match: self.redis_logger.warning(f'tracker yara: new match {self.item.get_id()}: {yara_match}') print(f'{self.item.get_id()}: {yara_match}') - except yara.TimeoutError as e: + except yara.TimeoutError: print(f'{self.item.get_id()}: yara scanning timed out') self.redis_logger.info(f'{self.item.get_id()}: yara scanning timed out') def yara_rules_match(self, data): tracker_uuid = data['namespace'] item_id = self.item.get_id() + + item = Item(item_id) item_source = self.item.get_source() - item_date = self.item.get_date() + + tracker = Tracker.Tracker(tracker_uuid) # Source Filtering - tracker_sources = Tracker.get_tracker_uuid_sources(tracker_uuid) + tracker_sources = tracker.get_sources() if tracker_sources and item_source not in tracker_sources: print(f'Source Filtering: {data["rule"]}') return yara.CALLBACK_CONTINUE - Tracker.add_tracked_item(tracker_uuid, item_id) + Tracker.add_tracked_item(tracker_uuid, item_id) # TODO # Tags - tags_to_add = Tracker.get_tracker_tags(tracker_uuid) - for tag in tags_to_add: - msg = '{};{}'.format(tag, item_id) + for tag in tracker.get_tags(): + msg = f'{tag};{item_id}' self.send_message_to_queue(msg, 'Tags') # Mails - mail_to_notify = Tracker.get_tracker_mails(tracker_uuid) - if mail_to_notify: - mail_subject = Tracker.get_email_subject(tracker_uuid) - mail_body = Tracker_Yara.mail_body_template.format(data['rule'], item_id, self.full_item_url, item_id) - for mail in mail_to_notify: - self.redis_logger.debug(f'Send Mail {mail_subject}') - print(f'Send Mail {mail_subject}') - NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body) + if tracker.mail_export(): + # TODO add matches + custom subjects + self.exporters['mail'].export(tracker, item) # Webhook - webhook_to_post = Term.get_term_webhook(tracker_uuid) - if webhook_to_post: - json_request = {"trackerId": tracker_uuid, - "itemId": item_id, - "itemURL": self.full_item_url + item_id, - "dataRule": data["rule"], - "itemSource": item_source, - "itemDate": item_date, - "tags": tags_to_add, - "emailNotification": f'{mail_to_notify}', - "trackerType": "yara" - } - try: - response = requests.post(webhook_to_post, json=json_request) - if response.status_code >= 400: - self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: {response.reason}") - except: - self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: Something went wrong") + if tracker.webhook_export(): + self.exporters['webhook'].export(tracker, item) return yara.CALLBACK_CONTINUE