From 2f52bdb9af8d4e043d0b90aabe9425b050198c27 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 2 Jun 2021 16:53:17 +0200 Subject: [PATCH] chg: [Tracker_Regex module] create module class + mv module --- bin/LAUNCH.sh | 7 +-- bin/RegexTracker.py | 87 --------------------------------- bin/packages/modules.cfg | 2 +- bin/trackers/Tracker_Regex.py | 92 +++++++++++++++++++++++++++++++++++ bin/trackers/Tracker_Term.py | 2 + configs/core.cfg.sample | 2 +- 6 files changed, 98 insertions(+), 94 deletions(-) delete mode 100755 bin/RegexTracker.py create mode 100755 bin/trackers/Tracker_Regex.py diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 5cbf9344..b9b3b03e 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -193,7 +193,8 @@ function launching_scripts { ################################## screen -S "Script_AIL" -X screen -t "Tracker_Term" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Term.py; read x" sleep 0.1 - + screen -S "Script_AIL" -X screen -t "Tracker_Regex" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Regex.py; read x" + sleep 0.1 screen -S "Script_AIL" -X screen -t "Tracker_Yara" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Yara.py; read x" sleep 0.1 @@ -212,16 +213,12 @@ function launching_scripts { sleep 0.1 screen -S "Script_AIL" -X screen -t "Duplicates" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Duplicates.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "CreditCards" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./CreditCards.py; read x" - sleep 0.1 screen -S "Script_AIL" -X screen -t "BankAccount" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./BankAccount.py; read x" sleep 0.1 screen -S "Script_AIL" -X screen -t "Mail" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mail.py; read x" sleep 0.1 screen -S "Script_AIL" -X screen -t "Urls" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Urls.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "RegexTracker" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./RegexTracker.py; read x" - sleep 0.1 screen -S "Script_AIL" -X screen -t "PgpDump" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./PgpDump.py; read x" sleep 0.1 screen -S "Script_AIL" -X screen -t "Cryptocurrency" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Cryptocurrencies.py; read x" diff --git a/bin/RegexTracker.py b/bin/RegexTracker.py deleted file mode 100755 index 904be623..00000000 --- a/bin/RegexTracker.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* -""" -This Module is used for regex tracking. -It processes every paste coming from the global module and test the regexs -supplied in the term webpage. - -""" -import os -import re -import sys -import time - -from Helper import Process -from pubsublogger import publisher - -import NotificationHelper - -from packages import Item -from packages import Term - -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) -import Tracker -import regex_helper - -full_item_url = "/object/item?id=" -mail_body_template = "AIL Framework,\nNew occurrence for term tracked regex: {}\nitem id: {}\nurl: {}{}" - -dict_regex_tracked = Term.get_regex_tracked_words_dict() -last_refresh = time.time() - -def new_term_found(term, term_type, item_id, item_date): - uuid_list = Term.get_term_uuid_list(term, 'regex') - print('new tracked term found: {} in {}'.format(term, item_id)) - - for term_uuid in uuid_list: - Term.add_tracked_item(term_uuid, item_id, item_date) - - tags_to_add = Term.get_term_tags(term_uuid) - for tag in tags_to_add: - msg = '{};{}'.format(tag, item_id) - p.populate_set_out(msg, 'Tags') - - mail_to_notify = Term.get_term_mails(term_uuid) - if mail_to_notify: - mail_subject = Tracker.get_email_subject(term_uuid) - mail_body = mail_body_template.format(term, item_id, full_item_url, item_id) - for mail in mail_to_notify: - NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body) - -if __name__ == "__main__": - publisher.port = 6380 - publisher.channel = "Script" - publisher.info("Script RegexTracker started") - - config_section = 'RegexTracker' - module_name = "RegexTracker" - p = Process(config_section) - max_execution_time = p.config.getint(config_section, "max_execution_time") - - full_item_url = p.config.get("Notifications", "ail_domain") + full_item_url - - redis_cache_key = regex_helper.generate_redis_cache_key(module_name) - - # Regex Frequency - while True: - - item_id = p.get_from_set() - - if item_id is not None: - - item_date = Item.get_item_date(item_id) - item_content = Item.get_item_content(item_id) - - for regex in dict_regex_tracked: - matched = regex_helper.regex_search(module_name, redis_cache_key, dict_regex_tracked[regex], item_id, item_content, max_time=max_execution_time) - if matched: - new_term_found(regex, 'regex', item_id, item_date) - - else: - time.sleep(5) - - # refresh Tracked term - if last_refresh < Term.get_tracked_term_last_updated_by_type('regex'): - dict_regex_tracked = Term.get_regex_tracked_words_dict() - last_refresh = time.time() - print('Tracked set refreshed') diff --git a/bin/packages/modules.cfg b/bin/packages/modules.cfg index 83ced250..542dc6dc 100644 --- a/bin/packages/modules.cfg +++ b/bin/packages/modules.cfg @@ -30,7 +30,7 @@ subscribe = Redis_D4_client subscribe = Redis_Global publish = Redis_Tags -[RegexTracker] +[Tracker_Regex] subscribe = Redis_Global publish = Redis_Tags diff --git a/bin/trackers/Tracker_Regex.py b/bin/trackers/Tracker_Regex.py new file mode 100755 index 00000000..82343661 --- /dev/null +++ b/bin/trackers/Tracker_Regex.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* +""" +The Tracker_Regex trackers module +=================== + +This Module is used for regex tracking. +It processes every item coming from the global module and test the regexs + +""" +import os +import re +import sys +import time + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from modules.abstract_module import AbstractModule +from packages.Item import Item +from packages import Term +from lib import Tracker +from lib import regex_helper + +import NotificationHelper + +class Tracker_Regex(AbstractModule): + + mail_body_template = "AIL Framework,\nNew occurrence for term tracked regex: {}\nitem id: {}\nurl: {}{}" + + """ + Tracker_Regex module for AIL framework + """ + def __init__(self): + super(Tracker_Regex, self).__init__() + + self.pending_seconds = 5 + + self.max_execution_time = self.process.config.getint(self.module_name, "max_execution_time") + + self.full_item_url = self.process.config.get("Notifications", "ail_domain") + "/object/item?id=" + + self.redis_cache_key = regex_helper.generate_redis_cache_key(self.module_name) + + # refresh Tracked term + self.dict_regex_tracked = Term.get_regex_tracked_words_dict() + self.last_refresh = time.time() + + self.redis_logger.info(f"Module: {self.module_name} Launched") + + def compute(self, item_id): + # refresh Tracked regex + if self.last_refresh < Term.get_tracked_term_last_updated_by_type('regex'): + self.dict_regex_tracked = Term.get_regex_tracked_words_dict() + self.last_refresh = time.time() + self.redis_logger.debug('Tracked word refreshed') + print('Tracked set refreshed') + + item = Item(item_id) + item_id = item.get_id() + item_date = item.get_date() + item_content = item.get_content() + + for regex in self.dict_regex_tracked: + matched = regex_helper.regex_search(self.module_name, self.redis_cache_key, self.dict_regex_tracked[regex], item_id, item_content, max_time=self.max_execution_time) + if matched: + self.new_term_found(regex, 'regex', item_id, item_date) + + def new_term_found(self, term, tracker_type, item_id, item_date): + uuid_list = Term.get_term_uuid_list(term, tracker_type) + print('new tracked regex found: {} in {}'.format(term, item_id)) + + for tracker_uuid in uuid_list: + Term.add_tracked_item(tracker_uuid, item_id, item_date) + + tags_to_add = Term.get_term_tags(tracker_uuid) + for tag in tags_to_add: + msg = '{};{}'.format(tag, item_id) + self.send_message_to_queue(msg, 'Tags') + + mail_to_notify = Term.get_term_mails(tracker_uuid) + if mail_to_notify: + mail_subject = Tracker.get_email_subject(tracker_uuid) + mail_body = Tracker_Regex.mail_body_template.format(term, item_id, self.full_item_url, item_id) + for mail in mail_to_notify: + NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body) + +if __name__ == "__main__": + + module = Tracker_Regex() + module.run() diff --git a/bin/trackers/Tracker_Term.py b/bin/trackers/Tracker_Term.py index 88a53a0a..89791e22 100755 --- a/bin/trackers/Tracker_Term.py +++ b/bin/trackers/Tracker_Term.py @@ -63,11 +63,13 @@ class Tracker_Term(AbstractModule): self.list_tracked_words = Term.get_tracked_words_list() self.last_refresh_word = time.time() self.redis_logger.debug('Tracked word refreshed') + print('Tracked word refreshed') if self.last_refresh_set < Term.get_tracked_term_last_updated_by_type('set'): self.set_tracked_words_list = Term.get_set_tracked_words_list() self.last_refresh_set = time.time() self.redis_logger.debug('Tracked set refreshed') + print('Tracked set refreshed') # Cast message as Item item = Item(item_id) diff --git a/configs/core.cfg.sample b/configs/core.cfg.sample index 89d6d22d..4bc871a0 100644 --- a/configs/core.cfg.sample +++ b/configs/core.cfg.sample @@ -119,7 +119,7 @@ default_unnamed_feed_name = unnamed_feeder [Tracker_Term] max_execution_time = 120 -[RegexTracker] +[Tracker_Regex] max_execution_time = 60 ##### Redis #####