Terrtia
2 달 전
No known key found for this signature in database
GPG 키 ID: 1E1B1F50D84613D0
6개의 변경된 파일과 98개의 추가작업 그리고 94개의 파일을 삭제
-
7bin/LAUNCH.sh
-
87bin/RegexTracker.py
-
2bin/packages/modules.cfg
-
92bin/trackers/Tracker_Regex.py
-
2bin/trackers/Tracker_Term.py
-
2configs/core.cfg.sample
@ -1,87 +0,0 @@ |
|||
#!/usr/bin/env python3 |
|||
# -*-coding:UTF-8 -* |
|||
""" |
|||
This Module is used for regex tracking. |
|||
It processes every paste coming from the global module and test the regexs |
|||
supplied in the term webpage. |
|||
|
|||
""" |
|||
import os |
|||
import re |
|||
import sys |
|||
import time |
|||
|
|||
from Helper import Process |
|||
from pubsublogger import publisher |
|||
|
|||
import NotificationHelper |
|||
|
|||
from packages import Item |
|||
from packages import Term |
|||
|
|||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) |
|||
import Tracker |
|||
import regex_helper |
|||
|
|||
full_item_url = "/object/item?id=" |
|||
mail_body_template = "AIL Framework,\nNew occurrence for term tracked regex: {}\nitem id: {}\nurl: {}{}" |
|||
|
|||
dict_regex_tracked = Term.get_regex_tracked_words_dict() |
|||
last_refresh = time.time() |
|||
|
|||
def new_term_found(term, term_type, item_id, item_date): |
|||
uuid_list = Term.get_term_uuid_list(term, 'regex') |
|||
print('new tracked term found: {} in {}'.format(term, item_id)) |
|||
|
|||
for term_uuid in uuid_list: |
|||
Term.add_tracked_item(term_uuid, item_id, item_date) |
|||
|
|||
tags_to_add = Term.get_term_tags(term_uuid) |
|||
for tag in tags_to_add: |
|||
msg = '{};{}'.format(tag, item_id) |
|||
p.populate_set_out(msg, 'Tags') |
|||
|
|||
mail_to_notify = Term.get_term_mails(term_uuid) |
|||
if mail_to_notify: |
|||
mail_subject = Tracker.get_email_subject(term_uuid) |
|||
mail_body = mail_body_template.format(term, item_id, full_item_url, item_id) |
|||
for mail in mail_to_notify: |
|||
NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body) |
|||
|
|||
if __name__ == "__main__": |
|||
publisher.port = 6380 |
|||
publisher.channel = "Script" |
|||
publisher.info("Script RegexTracker started") |
|||
|
|||
config_section = 'RegexTracker' |
|||
module_name = "RegexTracker" |
|||
p = Process(config_section) |
|||
max_execution_time = p.config.getint(config_section, "max_execution_time") |
|||
|
|||
full_item_url = p.config.get("Notifications", "ail_domain") + full_item_url |
|||
|
|||
redis_cache_key = regex_helper.generate_redis_cache_key(module_name) |
|||
|
|||
# Regex Frequency |
|||
while True: |
|||
|
|||
item_id = p.get_from_set() |
|||
|
|||
if item_id is not None: |
|||
|
|||
item_date = Item.get_item_date(item_id) |
|||
item_content = Item.get_item_content(item_id) |
|||
|
|||
for regex in dict_regex_tracked: |
|||
matched = regex_helper.regex_search(module_name, redis_cache_key, dict_regex_tracked[regex], item_id, item_content, max_time=max_execution_time) |
|||
if matched: |
|||
new_term_found(regex, 'regex', item_id, item_date) |
|||
|
|||
else: |
|||
time.sleep(5) |
|||
|
|||
# refresh Tracked term |
|||
if last_refresh < Term.get_tracked_term_last_updated_by_type('regex'): |
|||
dict_regex_tracked = Term.get_regex_tracked_words_dict() |
|||
last_refresh = time.time() |
|||
print('Tracked set refreshed') |
|||
@ -0,0 +1,92 @@ |
|||
#!/usr/bin/env python3 |
|||
# -*-coding:UTF-8 -* |
|||
""" |
|||
The Tracker_Regex trackers module |
|||
=================== |
|||
|
|||
This Module is used for regex tracking. |
|||
It processes every item coming from the global module and test the regexs |
|||
|
|||
""" |
|||
import os |
|||
import re |
|||
import sys |
|||
import time |
|||
|
|||
sys.path.append(os.environ['AIL_BIN']) |
|||
################################## |
|||
# Import Project packages |
|||
################################## |
|||
from modules.abstract_module import AbstractModule |
|||
from packages.Item import Item |
|||
from packages import Term |
|||
from lib import Tracker |
|||
from lib import regex_helper |
|||
|
|||
import NotificationHelper |
|||
|
|||
class Tracker_Regex(AbstractModule): |
|||
|
|||
mail_body_template = "AIL Framework,\nNew occurrence for term tracked regex: {}\nitem id: {}\nurl: {}{}" |
|||
|
|||
""" |
|||
Tracker_Regex module for AIL framework |
|||
""" |
|||
def __init__(self): |
|||
super(Tracker_Regex, self).__init__() |
|||
|
|||
self.pending_seconds = 5 |
|||
|
|||
self.max_execution_time = self.process.config.getint(self.module_name, "max_execution_time") |
|||
|
|||
self.full_item_url = self.process.config.get("Notifications", "ail_domain") + "/object/item?id=" |
|||
|
|||
self.redis_cache_key = regex_helper.generate_redis_cache_key(self.module_name) |
|||
|
|||
# refresh Tracked term |
|||
self.dict_regex_tracked = Term.get_regex_tracked_words_dict() |
|||
self.last_refresh = time.time() |
|||
|
|||
self.redis_logger.info(f"Module: {self.module_name} Launched") |
|||
|
|||
def compute(self, item_id): |
|||
# refresh Tracked regex |
|||
if self.last_refresh < Term.get_tracked_term_last_updated_by_type('regex'): |
|||
self.dict_regex_tracked = Term.get_regex_tracked_words_dict() |
|||
self.last_refresh = time.time() |
|||
self.redis_logger.debug('Tracked word refreshed') |
|||
print('Tracked set refreshed') |
|||
|
|||
item = Item(item_id) |
|||
item_id = item.get_id() |
|||
item_date = item.get_date() |
|||
item_content = item.get_content() |
|||
|
|||
for regex in self.dict_regex_tracked: |
|||
matched = regex_helper.regex_search(self.module_name, self.redis_cache_key, self.dict_regex_tracked[regex], item_id, item_content, max_time=self.max_execution_time) |
|||
if matched: |
|||
self.new_term_found(regex, 'regex', item_id, item_date) |
|||
|
|||
def new_term_found(self, term, tracker_type, item_id, item_date): |
|||
uuid_list = Term.get_term_uuid_list(term, tracker_type) |
|||
print('new tracked regex found: {} in {}'.format(term, item_id)) |
|||
|
|||
for tracker_uuid in uuid_list: |
|||
Term.add_tracked_item(tracker_uuid, item_id, item_date) |
|||
|
|||
tags_to_add = Term.get_term_tags(tracker_uuid) |
|||
for tag in tags_to_add: |
|||
msg = '{};{}'.format(tag, item_id) |
|||
self.send_message_to_queue(msg, 'Tags') |
|||
|
|||
mail_to_notify = Term.get_term_mails(tracker_uuid) |
|||
if mail_to_notify: |
|||
mail_subject = Tracker.get_email_subject(tracker_uuid) |
|||
mail_body = Tracker_Regex.mail_body_template.format(term, item_id, self.full_item_url, item_id) |
|||
for mail in mail_to_notify: |
|||
NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body) |
|||
|
|||
if __name__ == "__main__": |
|||
|
|||
module = Tracker_Regex() |
|||
module.run() |
|||
쓰기
미리보기
불러오는 중...
취소
저장
Reference in new issue