chg: [Tracker_Regex module] create module class + mv module

pull/569/head
Terrtia 2021-06-02 16:53:17 +02:00
parent 5acb583701
commit 2f52bdb9af
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
6 changed files with 98 additions and 94 deletions

View File

@ -193,7 +193,8 @@ function launching_scripts {
################################## ##################################
screen -S "Script_AIL" -X screen -t "Tracker_Term" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Term.py; read x" screen -S "Script_AIL" -X screen -t "Tracker_Term" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Term.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Tracker_Regex" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Regex.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Tracker_Yara" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Yara.py; read x" screen -S "Script_AIL" -X screen -t "Tracker_Yara" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Yara.py; read x"
sleep 0.1 sleep 0.1
@ -212,16 +213,12 @@ function launching_scripts {
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Duplicates" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Duplicates.py; read x" screen -S "Script_AIL" -X screen -t "Duplicates" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Duplicates.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "CreditCards" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./CreditCards.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "BankAccount" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./BankAccount.py; read x" screen -S "Script_AIL" -X screen -t "BankAccount" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./BankAccount.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Mail" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mail.py; read x" screen -S "Script_AIL" -X screen -t "Mail" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mail.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Urls" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Urls.py; read x" screen -S "Script_AIL" -X screen -t "Urls" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Urls.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "RegexTracker" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./RegexTracker.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "PgpDump" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./PgpDump.py; read x" screen -S "Script_AIL" -X screen -t "PgpDump" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./PgpDump.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Cryptocurrency" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Cryptocurrencies.py; read x" screen -S "Script_AIL" -X screen -t "Cryptocurrency" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Cryptocurrencies.py; read x"

View File

@ -1,87 +0,0 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
This Module is used for regex tracking.
It processes every paste coming from the global module and test the regexs
supplied in the term webpage.
"""
import os
import re
import sys
import time
from Helper import Process
from pubsublogger import publisher
import NotificationHelper
from packages import Item
from packages import Term
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
import Tracker
import regex_helper
full_item_url = "/object/item?id="
mail_body_template = "AIL Framework,\nNew occurrence for term tracked regex: {}\nitem id: {}\nurl: {}{}"
dict_regex_tracked = Term.get_regex_tracked_words_dict()
last_refresh = time.time()
def new_term_found(term, term_type, item_id, item_date):
uuid_list = Term.get_term_uuid_list(term, 'regex')
print('new tracked term found: {} in {}'.format(term, item_id))
for term_uuid in uuid_list:
Term.add_tracked_item(term_uuid, item_id, item_date)
tags_to_add = Term.get_term_tags(term_uuid)
for tag in tags_to_add:
msg = '{};{}'.format(tag, item_id)
p.populate_set_out(msg, 'Tags')
mail_to_notify = Term.get_term_mails(term_uuid)
if mail_to_notify:
mail_subject = Tracker.get_email_subject(term_uuid)
mail_body = mail_body_template.format(term, item_id, full_item_url, item_id)
for mail in mail_to_notify:
NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
if __name__ == "__main__":
publisher.port = 6380
publisher.channel = "Script"
publisher.info("Script RegexTracker started")
config_section = 'RegexTracker'
module_name = "RegexTracker"
p = Process(config_section)
max_execution_time = p.config.getint(config_section, "max_execution_time")
full_item_url = p.config.get("Notifications", "ail_domain") + full_item_url
redis_cache_key = regex_helper.generate_redis_cache_key(module_name)
# Regex Frequency
while True:
item_id = p.get_from_set()
if item_id is not None:
item_date = Item.get_item_date(item_id)
item_content = Item.get_item_content(item_id)
for regex in dict_regex_tracked:
matched = regex_helper.regex_search(module_name, redis_cache_key, dict_regex_tracked[regex], item_id, item_content, max_time=max_execution_time)
if matched:
new_term_found(regex, 'regex', item_id, item_date)
else:
time.sleep(5)
# refresh Tracked term
if last_refresh < Term.get_tracked_term_last_updated_by_type('regex'):
dict_regex_tracked = Term.get_regex_tracked_words_dict()
last_refresh = time.time()
print('Tracked set refreshed')

View File

@ -30,7 +30,7 @@ subscribe = Redis_D4_client
subscribe = Redis_Global subscribe = Redis_Global
publish = Redis_Tags publish = Redis_Tags
[RegexTracker] [Tracker_Regex]
subscribe = Redis_Global subscribe = Redis_Global
publish = Redis_Tags publish = Redis_Tags

92
bin/trackers/Tracker_Regex.py Executable file
View File

@ -0,0 +1,92 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
The Tracker_Regex trackers module
===================
This Module is used for regex tracking.
It processes every item coming from the global module and test the regexs
"""
import os
import re
import sys
import time
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from modules.abstract_module import AbstractModule
from packages.Item import Item
from packages import Term
from lib import Tracker
from lib import regex_helper
import NotificationHelper
class Tracker_Regex(AbstractModule):
mail_body_template = "AIL Framework,\nNew occurrence for term tracked regex: {}\nitem id: {}\nurl: {}{}"
"""
Tracker_Regex module for AIL framework
"""
def __init__(self):
super(Tracker_Regex, self).__init__()
self.pending_seconds = 5
self.max_execution_time = self.process.config.getint(self.module_name, "max_execution_time")
self.full_item_url = self.process.config.get("Notifications", "ail_domain") + "/object/item?id="
self.redis_cache_key = regex_helper.generate_redis_cache_key(self.module_name)
# refresh Tracked term
self.dict_regex_tracked = Term.get_regex_tracked_words_dict()
self.last_refresh = time.time()
self.redis_logger.info(f"Module: {self.module_name} Launched")
def compute(self, item_id):
# refresh Tracked regex
if self.last_refresh < Term.get_tracked_term_last_updated_by_type('regex'):
self.dict_regex_tracked = Term.get_regex_tracked_words_dict()
self.last_refresh = time.time()
self.redis_logger.debug('Tracked word refreshed')
print('Tracked set refreshed')
item = Item(item_id)
item_id = item.get_id()
item_date = item.get_date()
item_content = item.get_content()
for regex in self.dict_regex_tracked:
matched = regex_helper.regex_search(self.module_name, self.redis_cache_key, self.dict_regex_tracked[regex], item_id, item_content, max_time=self.max_execution_time)
if matched:
self.new_term_found(regex, 'regex', item_id, item_date)
def new_term_found(self, term, tracker_type, item_id, item_date):
uuid_list = Term.get_term_uuid_list(term, tracker_type)
print('new tracked regex found: {} in {}'.format(term, item_id))
for tracker_uuid in uuid_list:
Term.add_tracked_item(tracker_uuid, item_id, item_date)
tags_to_add = Term.get_term_tags(tracker_uuid)
for tag in tags_to_add:
msg = '{};{}'.format(tag, item_id)
self.send_message_to_queue(msg, 'Tags')
mail_to_notify = Term.get_term_mails(tracker_uuid)
if mail_to_notify:
mail_subject = Tracker.get_email_subject(tracker_uuid)
mail_body = Tracker_Regex.mail_body_template.format(term, item_id, self.full_item_url, item_id)
for mail in mail_to_notify:
NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
if __name__ == "__main__":
module = Tracker_Regex()
module.run()

View File

@ -63,11 +63,13 @@ class Tracker_Term(AbstractModule):
self.list_tracked_words = Term.get_tracked_words_list() self.list_tracked_words = Term.get_tracked_words_list()
self.last_refresh_word = time.time() self.last_refresh_word = time.time()
self.redis_logger.debug('Tracked word refreshed') self.redis_logger.debug('Tracked word refreshed')
print('Tracked word refreshed')
if self.last_refresh_set < Term.get_tracked_term_last_updated_by_type('set'): if self.last_refresh_set < Term.get_tracked_term_last_updated_by_type('set'):
self.set_tracked_words_list = Term.get_set_tracked_words_list() self.set_tracked_words_list = Term.get_set_tracked_words_list()
self.last_refresh_set = time.time() self.last_refresh_set = time.time()
self.redis_logger.debug('Tracked set refreshed') self.redis_logger.debug('Tracked set refreshed')
print('Tracked set refreshed')
# Cast message as Item # Cast message as Item
item = Item(item_id) item = Item(item_id)

View File

@ -119,7 +119,7 @@ default_unnamed_feed_name = unnamed_feeder
[Tracker_Term] [Tracker_Term]
max_execution_time = 120 max_execution_time = 120
[RegexTracker] [Tracker_Regex]
max_execution_time = 60 max_execution_time = 60
##### Redis ##### ##### Redis #####