mirror of https://github.com/CIRCL/AIL-framework
chg: [Tracker_Regex module] create module class + mv module
Terrtia
parent
5acb583701
commit
2f52bdb9af
|
|
@ -193,7 +193,8 @@ function launching_scripts {
|
|||
##################################
|
||||
screen -S "Script_AIL" -X screen -t "Tracker_Term" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Term.py; read x"
|
||||
sleep 0.1
|
||||
|
||||
screen -S "Script_AIL" -X screen -t "Tracker_Regex" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Regex.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Tracker_Yara" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Yara.py; read x"
|
||||
sleep 0.1
|
||||
|
||||
|
|
@ -212,16 +213,12 @@ function launching_scripts {
|
|||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Duplicates" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Duplicates.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "CreditCards" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./CreditCards.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "BankAccount" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./BankAccount.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Mail" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mail.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Urls" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Urls.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "RegexTracker" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./RegexTracker.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "PgpDump" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./PgpDump.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Cryptocurrency" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Cryptocurrencies.py; read x"
|
||||
|
|
|
|||
|
|
@ -1,87 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
This Module is used for regex tracking.
|
||||
It processes every paste coming from the global module and test the regexs
|
||||
supplied in the term webpage.
|
||||
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
|
||||
from Helper import Process
|
||||
from pubsublogger import publisher
|
||||
|
||||
import NotificationHelper
|
||||
|
||||
from packages import Item
|
||||
from packages import Term
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
|
||||
import Tracker
|
||||
import regex_helper
|
||||
|
||||
full_item_url = "/object/item?id="
|
||||
mail_body_template = "AIL Framework,\nNew occurrence for term tracked regex: {}\nitem id: {}\nurl: {}{}"
|
||||
|
||||
dict_regex_tracked = Term.get_regex_tracked_words_dict()
|
||||
last_refresh = time.time()
|
||||
|
||||
def new_term_found(term, term_type, item_id, item_date):
|
||||
uuid_list = Term.get_term_uuid_list(term, 'regex')
|
||||
print('new tracked term found: {} in {}'.format(term, item_id))
|
||||
|
||||
for term_uuid in uuid_list:
|
||||
Term.add_tracked_item(term_uuid, item_id, item_date)
|
||||
|
||||
tags_to_add = Term.get_term_tags(term_uuid)
|
||||
for tag in tags_to_add:
|
||||
msg = '{};{}'.format(tag, item_id)
|
||||
p.populate_set_out(msg, 'Tags')
|
||||
|
||||
mail_to_notify = Term.get_term_mails(term_uuid)
|
||||
if mail_to_notify:
|
||||
mail_subject = Tracker.get_email_subject(term_uuid)
|
||||
mail_body = mail_body_template.format(term, item_id, full_item_url, item_id)
|
||||
for mail in mail_to_notify:
|
||||
NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
|
||||
|
||||
if __name__ == "__main__":
|
||||
publisher.port = 6380
|
||||
publisher.channel = "Script"
|
||||
publisher.info("Script RegexTracker started")
|
||||
|
||||
config_section = 'RegexTracker'
|
||||
module_name = "RegexTracker"
|
||||
p = Process(config_section)
|
||||
max_execution_time = p.config.getint(config_section, "max_execution_time")
|
||||
|
||||
full_item_url = p.config.get("Notifications", "ail_domain") + full_item_url
|
||||
|
||||
redis_cache_key = regex_helper.generate_redis_cache_key(module_name)
|
||||
|
||||
# Regex Frequency
|
||||
while True:
|
||||
|
||||
item_id = p.get_from_set()
|
||||
|
||||
if item_id is not None:
|
||||
|
||||
item_date = Item.get_item_date(item_id)
|
||||
item_content = Item.get_item_content(item_id)
|
||||
|
||||
for regex in dict_regex_tracked:
|
||||
matched = regex_helper.regex_search(module_name, redis_cache_key, dict_regex_tracked[regex], item_id, item_content, max_time=max_execution_time)
|
||||
if matched:
|
||||
new_term_found(regex, 'regex', item_id, item_date)
|
||||
|
||||
else:
|
||||
time.sleep(5)
|
||||
|
||||
# refresh Tracked term
|
||||
if last_refresh < Term.get_tracked_term_last_updated_by_type('regex'):
|
||||
dict_regex_tracked = Term.get_regex_tracked_words_dict()
|
||||
last_refresh = time.time()
|
||||
print('Tracked set refreshed')
|
||||
|
|
@ -30,7 +30,7 @@ subscribe = Redis_D4_client
|
|||
subscribe = Redis_Global
|
||||
publish = Redis_Tags
|
||||
|
||||
[RegexTracker]
|
||||
[Tracker_Regex]
|
||||
subscribe = Redis_Global
|
||||
publish = Redis_Tags
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,92 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
The Tracker_Regex trackers module
|
||||
===================
|
||||
|
||||
This Module is used for regex tracking.
|
||||
It processes every item coming from the global module and test the regexs
|
||||
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from modules.abstract_module import AbstractModule
|
||||
from packages.Item import Item
|
||||
from packages import Term
|
||||
from lib import Tracker
|
||||
from lib import regex_helper
|
||||
|
||||
import NotificationHelper
|
||||
|
||||
class Tracker_Regex(AbstractModule):
|
||||
|
||||
mail_body_template = "AIL Framework,\nNew occurrence for term tracked regex: {}\nitem id: {}\nurl: {}{}"
|
||||
|
||||
"""
|
||||
Tracker_Regex module for AIL framework
|
||||
"""
|
||||
def __init__(self):
|
||||
super(Tracker_Regex, self).__init__()
|
||||
|
||||
self.pending_seconds = 5
|
||||
|
||||
self.max_execution_time = self.process.config.getint(self.module_name, "max_execution_time")
|
||||
|
||||
self.full_item_url = self.process.config.get("Notifications", "ail_domain") + "/object/item?id="
|
||||
|
||||
self.redis_cache_key = regex_helper.generate_redis_cache_key(self.module_name)
|
||||
|
||||
# refresh Tracked term
|
||||
self.dict_regex_tracked = Term.get_regex_tracked_words_dict()
|
||||
self.last_refresh = time.time()
|
||||
|
||||
self.redis_logger.info(f"Module: {self.module_name} Launched")
|
||||
|
||||
def compute(self, item_id):
|
||||
# refresh Tracked regex
|
||||
if self.last_refresh < Term.get_tracked_term_last_updated_by_type('regex'):
|
||||
self.dict_regex_tracked = Term.get_regex_tracked_words_dict()
|
||||
self.last_refresh = time.time()
|
||||
self.redis_logger.debug('Tracked word refreshed')
|
||||
print('Tracked set refreshed')
|
||||
|
||||
item = Item(item_id)
|
||||
item_id = item.get_id()
|
||||
item_date = item.get_date()
|
||||
item_content = item.get_content()
|
||||
|
||||
for regex in self.dict_regex_tracked:
|
||||
matched = regex_helper.regex_search(self.module_name, self.redis_cache_key, self.dict_regex_tracked[regex], item_id, item_content, max_time=self.max_execution_time)
|
||||
if matched:
|
||||
self.new_term_found(regex, 'regex', item_id, item_date)
|
||||
|
||||
def new_term_found(self, term, tracker_type, item_id, item_date):
|
||||
uuid_list = Term.get_term_uuid_list(term, tracker_type)
|
||||
print('new tracked regex found: {} in {}'.format(term, item_id))
|
||||
|
||||
for tracker_uuid in uuid_list:
|
||||
Term.add_tracked_item(tracker_uuid, item_id, item_date)
|
||||
|
||||
tags_to_add = Term.get_term_tags(tracker_uuid)
|
||||
for tag in tags_to_add:
|
||||
msg = '{};{}'.format(tag, item_id)
|
||||
self.send_message_to_queue(msg, 'Tags')
|
||||
|
||||
mail_to_notify = Term.get_term_mails(tracker_uuid)
|
||||
if mail_to_notify:
|
||||
mail_subject = Tracker.get_email_subject(tracker_uuid)
|
||||
mail_body = Tracker_Regex.mail_body_template.format(term, item_id, self.full_item_url, item_id)
|
||||
for mail in mail_to_notify:
|
||||
NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
module = Tracker_Regex()
|
||||
module.run()
|
||||
|
|
@ -63,11 +63,13 @@ class Tracker_Term(AbstractModule):
|
|||
self.list_tracked_words = Term.get_tracked_words_list()
|
||||
self.last_refresh_word = time.time()
|
||||
self.redis_logger.debug('Tracked word refreshed')
|
||||
print('Tracked word refreshed')
|
||||
|
||||
if self.last_refresh_set < Term.get_tracked_term_last_updated_by_type('set'):
|
||||
self.set_tracked_words_list = Term.get_set_tracked_words_list()
|
||||
self.last_refresh_set = time.time()
|
||||
self.redis_logger.debug('Tracked set refreshed')
|
||||
print('Tracked set refreshed')
|
||||
|
||||
# Cast message as Item
|
||||
item = Item(item_id)
|
||||
|
|
|
|||
|
|
@ -119,7 +119,7 @@ default_unnamed_feed_name = unnamed_feeder
|
|||
[Tracker_Term]
|
||||
max_execution_time = 120
|
||||
|
||||
[RegexTracker]
|
||||
[Tracker_Regex]
|
||||
max_execution_time = 60
|
||||
|
||||
##### Redis #####
|
||||
|
|
|
|||
Loading…
Reference in New Issue