chg: [Tracker_Regex module] create module class + mv module

bin/LAUNCH.sh

@@ -193,7 +193,8 @@ function launching_scripts {
     ##################################
     screen -S "Script_AIL" -X screen -t "Tracker_Term" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Term.py; read x"
     sleep 0.1
-
+    screen -S "Script_AIL" -X screen -t "Tracker_Regex" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Regex.py; read x"
+    sleep 0.1
     screen -S "Script_AIL" -X screen -t "Tracker_Yara" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Yara.py; read x"
     sleep 0.1
 
@@ -212,16 +213,12 @@ function launching_scripts {
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "Duplicates" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Duplicates.py; read x"
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "CreditCards" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./CreditCards.py; read x"
-    sleep 0.1
     screen -S "Script_AIL" -X screen -t "BankAccount" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./BankAccount.py; read x"
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "Mail" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mail.py; read x"
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "Urls" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Urls.py; read x"
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "RegexTracker" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./RegexTracker.py; read x"
-    sleep 0.1
     screen -S "Script_AIL" -X screen -t "PgpDump" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./PgpDump.py; read x"
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "Cryptocurrency" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Cryptocurrencies.py; read x"

bin/RegexTracker.py

@@ -1,87 +0,0 @@
-#!/usr/bin/env python3
-# -*-coding:UTF-8 -*
-"""
-This Module is used for regex tracking.
-It processes every paste coming from the global module and test the regexs
-supplied in  the term webpage.
-
-"""
-import os
-import re
-import sys
-import time
-
-from Helper import Process
-from pubsublogger import publisher
-
-import NotificationHelper
-
-from packages import Item
-from packages import Term
-
-sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
-import Tracker
-import regex_helper
-
-full_item_url = "/object/item?id="
-mail_body_template = "AIL Framework,\nNew occurrence for term tracked regex: {}\nitem id: {}\nurl: {}{}"
-
-dict_regex_tracked = Term.get_regex_tracked_words_dict()
-last_refresh = time.time()
-
-def new_term_found(term, term_type, item_id, item_date):
-    uuid_list = Term.get_term_uuid_list(term, 'regex')
-    print('new tracked term found: {} in {}'.format(term, item_id))
-
-    for term_uuid in uuid_list:
-        Term.add_tracked_item(term_uuid, item_id, item_date)
-
-        tags_to_add = Term.get_term_tags(term_uuid)
-        for tag in tags_to_add:
-            msg = '{};{}'.format(tag, item_id)
-            p.populate_set_out(msg, 'Tags')
-
-        mail_to_notify = Term.get_term_mails(term_uuid)
-        if mail_to_notify:
-            mail_subject = Tracker.get_email_subject(term_uuid)
-            mail_body = mail_body_template.format(term, item_id, full_item_url, item_id)
-        for mail in mail_to_notify:
-            NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
-
-if __name__ == "__main__":
-    publisher.port = 6380
-    publisher.channel = "Script"
-    publisher.info("Script RegexTracker started")
-
-    config_section = 'RegexTracker'
-    module_name = "RegexTracker"
-    p = Process(config_section)
-    max_execution_time = p.config.getint(config_section, "max_execution_time")
-
-    full_item_url = p.config.get("Notifications", "ail_domain") + full_item_url
-
-    redis_cache_key = regex_helper.generate_redis_cache_key(module_name)
-
-    # Regex Frequency
-    while True:
-
-        item_id = p.get_from_set()
-
-        if item_id is not None:
-
-            item_date = Item.get_item_date(item_id)
-            item_content = Item.get_item_content(item_id)
-
-            for regex in dict_regex_tracked:
-                matched = regex_helper.regex_search(module_name, redis_cache_key, dict_regex_tracked[regex], item_id, item_content, max_time=max_execution_time)
-                if matched:
-                    new_term_found(regex, 'regex', item_id, item_date)
-
-        else:
-            time.sleep(5)
-
-        # refresh Tracked term
-        if last_refresh < Term.get_tracked_term_last_updated_by_type('regex'):
-            dict_regex_tracked = Term.get_regex_tracked_words_dict()
-            last_refresh = time.time()
-            print('Tracked set refreshed')

bin/packages/modules.cfg

@@ -30,7 +30,7 @@ subscribe = Redis_D4_client
 subscribe = Redis_Global
 publish = Redis_Tags
 
-[RegexTracker]
+[Tracker_Regex]
 subscribe = Redis_Global
 publish = Redis_Tags
 

bin/trackers/Tracker_Regex.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+"""
+The Tracker_Regex trackers module
+===================
+
+This Module is used for regex tracking.
+It processes every item coming from the global module and test the regexs
+
+"""
+import os
+import re
+import sys
+import time
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from modules.abstract_module import AbstractModule
+from packages.Item import Item
+from packages import Term
+from lib import Tracker
+from lib import regex_helper
+
+import NotificationHelper
+
+class Tracker_Regex(AbstractModule):
+
+    mail_body_template = "AIL Framework,\nNew occurrence for term tracked regex: {}\nitem id: {}\nurl: {}{}"
+
+    """
+    Tracker_Regex module for AIL framework
+    """
+    def __init__(self):
+        super(Tracker_Regex, self).__init__()
+
+        self.pending_seconds = 5
+
+        self.max_execution_time = self.process.config.getint(self.module_name, "max_execution_time")
+
+        self.full_item_url = self.process.config.get("Notifications", "ail_domain") + "/object/item?id="
+
+        self.redis_cache_key = regex_helper.generate_redis_cache_key(self.module_name)
+
+        # refresh Tracked term
+        self.dict_regex_tracked = Term.get_regex_tracked_words_dict()
+        self.last_refresh = time.time()
+
+        self.redis_logger.info(f"Module: {self.module_name} Launched")
+
+    def compute(self, item_id):
+        # refresh Tracked regex
+        if self.last_refresh < Term.get_tracked_term_last_updated_by_type('regex'):
+            self.dict_regex_tracked = Term.get_regex_tracked_words_dict()
+            self.last_refresh = time.time()
+            self.redis_logger.debug('Tracked word refreshed')
+            print('Tracked set refreshed')
+
+        item = Item(item_id)
+        item_id = item.get_id()
+        item_date = item.get_date()
+        item_content = item.get_content()
+
+        for regex in self.dict_regex_tracked:
+            matched = regex_helper.regex_search(self.module_name, self.redis_cache_key, self.dict_regex_tracked[regex], item_id, item_content, max_time=self.max_execution_time)
+            if matched:
+                self.new_term_found(regex, 'regex', item_id, item_date)
+
+    def new_term_found(self, term, tracker_type, item_id, item_date):
+        uuid_list = Term.get_term_uuid_list(term, tracker_type)
+        print('new tracked regex found: {} in {}'.format(term, item_id))
+
+        for tracker_uuid in uuid_list:
+            Term.add_tracked_item(tracker_uuid, item_id, item_date)
+
+            tags_to_add = Term.get_term_tags(tracker_uuid)
+            for tag in tags_to_add:
+                msg = '{};{}'.format(tag, item_id)
+                self.send_message_to_queue(msg, 'Tags')
+
+            mail_to_notify = Term.get_term_mails(tracker_uuid)
+            if mail_to_notify:
+                mail_subject = Tracker.get_email_subject(tracker_uuid)
+                mail_body = Tracker_Regex.mail_body_template.format(term, item_id, self.full_item_url, item_id)
+            for mail in mail_to_notify:
+                NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
+
+if __name__ == "__main__":
+
+    module = Tracker_Regex()
+    module.run()

bin/trackers/Tracker_Term.py

@@ -63,11 +63,13 @@ class Tracker_Term(AbstractModule):
             self.list_tracked_words = Term.get_tracked_words_list()
             self.last_refresh_word = time.time()
             self.redis_logger.debug('Tracked word refreshed')
+            print('Tracked word refreshed')
 
         if self.last_refresh_set < Term.get_tracked_term_last_updated_by_type('set'):
             self.set_tracked_words_list = Term.get_set_tracked_words_list()
             self.last_refresh_set = time.time()
             self.redis_logger.debug('Tracked set refreshed')
+            print('Tracked set refreshed')
 
         # Cast message as Item
         item = Item(item_id)

configs/core.cfg.sample

@@ -119,7 +119,7 @@ default_unnamed_feed_name = unnamed_feeder
 [Tracker_Term]
 max_execution_time = 120
 
-[RegexTracker]
+[Tracker_Regex]
 max_execution_time = 60
 
 ##### Redis #####