CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

fix: [tracker] fix exporters

pull/594/head
Terrtia 2023-03-30 14:58:55 +02:00
parent 126ecb2e39
commit 5f150489b6
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
4 changed files with 63 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
bin/trackers/Tracker_Regex.py
View File

@ -25,9 +25,6 @@ from exporter.MailExporter import MailExporterTracker
from exporter.WebHookExporter import WebHookExporterTracker from exporter.WebHookExporter import WebHookExporterTracker
class Tracker_Regex(AbstractModule): class Tracker_Regex(AbstractModule):
mail_body_template = "AIL Framework,\nNew occurrence for tracked regex: {}\nitem id: {}\nurl: {}{}"
""" """
Tracker_Regex module for AIL framework Tracker_Regex module for AIL framework
""" """
@ -66,18 +63,12 @@ class Tracker_Regex(AbstractModule):
if matched: if matched:
self.new_tracker_found(regex, 'regex', item) self.new_tracker_found(regex, 'regex', item)
# match = self.regex_finditer(self.dict_regex_tracked[regex], item_id, content)
# if match:
# self.new_tracker_found(regex, 'regex', item)
def new_tracker_found(self, tracker_name, tracker_type, item): def new_tracker_found(self, tracker_name, tracker_type, item):
uuid_list = Tracker.get_tracker_uuid_list(tracker_name, tracker_type) uuid_list = Tracker.get_tracker_uuid_list(tracker_name, tracker_type)
item_id = item.get_id() item_id = item.get_id()
# date = item.get_date() # date = item.get_date()
item_source = item.get_source() item_source = item.get_source()
print(f'new tracked regex found: {tracker_name} in {item_id}')
self.redis_logger.warning(f'new tracked regex found: {tracker_name} in {item_id}')
for tracker_uuid in uuid_list: for tracker_uuid in uuid_list:
tracker = Tracker.Tracker(tracker_uuid) tracker = Tracker.Tracker(tracker_uuid)
@ -87,7 +78,10 @@ class Tracker_Regex(AbstractModule):
if tracker_sources and item_source not in tracker_sources: if tracker_sources and item_source not in tracker_sources:
continue continue
Tracker.add_tracked_item(tracker_uuid, item_id) # TODO print(f'new tracked regex found: {tracker_name} in {item_id}')
self.redis_logger.warning(f'new tracked regex found: {tracker_name} in {item_id}')
# TODO
Tracker.add_tracked_item(tracker_uuid, item_id)
for tag in tracker.get_tags(): for tag in tracker.get_tags():
msg = f'{tag};{item_id}' msg = f'{tag};{item_id}'

80
bin/trackers/Tracker_Term.py
View File

@ -13,7 +13,6 @@ import os
import sys import sys
import time import time
import signal import signal
import requests
sys.path.append(os.environ['AIL_BIN']) sys.path.append(os.environ['AIL_BIN'])
@ -21,11 +20,13 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages # Import Project packages
################################## ##################################
from modules.abstract_module import AbstractModule from modules.abstract_module import AbstractModule
import NotificationHelper
from lib.objects.Items import Item from lib.objects.Items import Item
from packages import Term from packages import Term
from lib import Tracker from lib import Tracker
from exporter.MailExporter import MailExporterTracker
from exporter.WebHookExporter import WebHookExporterTracker
class TimeoutException(Exception): class TimeoutException(Exception):
pass pass
@ -38,8 +39,6 @@ signal.signal(signal.SIGALRM, timeout_handler)
class Tracker_Term(AbstractModule): class Tracker_Term(AbstractModule):
mail_body_template = "AIL Framework,\nNew occurrence for tracked term: {}\nitem id: {}\nurl: {}{}"
""" """
Tracker_Term module for AIL framework Tracker_Term module for AIL framework
""" """
@ -51,14 +50,16 @@ class Tracker_Term(AbstractModule):
self.max_execution_time = self.process.config.getint('Tracker_Term', "max_execution_time") self.max_execution_time = self.process.config.getint('Tracker_Term', "max_execution_time")
self.full_item_url = self.process.config.get("Notifications", "ail_domain") + "/object/item?id="
# loads tracked words # loads tracked words
self.list_tracked_words = Term.get_tracked_words_list() self.list_tracked_words = Term.get_tracked_words_list()
self.last_refresh_word = time.time() self.last_refresh_word = time.time()
self.set_tracked_words_list = Term.get_set_tracked_words_list() self.set_tracked_words_list = Term.get_set_tracked_words_list()
self.last_refresh_set = time.time() self.last_refresh_set = time.time()
# Exporter
self.exporters = {'mail': MailExporterTracker(),
'webhook': WebHookExporterTracker()}
self.redis_logger.info(f"Module: {self.module_name} Launched") self.redis_logger.info(f"Module: {self.module_name} Launched")
def compute(self, item_id, item_content=None): def compute(self, item_id, item_content=None):
@ -77,7 +78,6 @@ class Tracker_Term(AbstractModule):
# Cast message as Item # Cast message as Item
item = Item(item_id) item = Item(item_id)
item_date = item.get_date()
if not item_content: if not item_content:
item_content = item.get_content() item_content = item.get_content()
@ -115,52 +115,38 @@ class Tracker_Term(AbstractModule):
if nb_uniq_word >= nb_words_threshold: if nb_uniq_word >= nb_words_threshold:
self.new_term_found(word_set, 'set', item) self.new_term_found(word_set, 'set', item)
def new_term_found(self, term, term_type, item): def new_term_found(self, tracker_name, tracker_type, item):
uuid_list = Term.get_term_uuid_list(term, term_type) uuid_list = Tracker.get_tracker_uuid_list(tracker_name, tracker_type)
item_id = item.get_id() item_id = item.get_id()
item_date = item.get_date()
item_source = item.get_source() item_source = item.get_source()
self.redis_logger.warning(f'new tracked term found: {term} in {item_id}')
print(f'new tracked term found: {term} in {item_id}')
for term_uuid in uuid_list:
tracker_sources = Tracker.get_tracker_uuid_sources(term_uuid)
if not tracker_sources or item_source in tracker_sources:
Tracker.add_tracked_item(term_uuid, item_id)
tags_to_add = Term.get_term_tags(term_uuid) for tracker_uuid in uuid_list:
for tag in tags_to_add: tracker = Tracker.Tracker(tracker_uuid)
msg = '{};{}'.format(tag, item_id)
self.send_message_to_queue(msg, 'Tags')
mail_to_notify = Term.get_term_mails(term_uuid) # Source Filtering
if mail_to_notify: tracker_sources = tracker.get_sources()
mail_subject = Tracker.get_email_subject(term_uuid) if tracker_sources and item_source not in tracker_sources:
mail_body = Tracker_Term.mail_body_template.format(term, item_id, self.full_item_url, item_id) continue
for mail in mail_to_notify:
self.redis_logger.debug(f'Send Mail {mail_subject}')
print(f'S print(item_content)end Mail {mail_subject}')
NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
# Webhook print(f'new tracked term found: {tracker_name} in {item_id}')
webhook_to_post = Term.get_term_webhook(term_uuid) self.redis_logger.warning(f'new tracked term found: {tracker_name} in {item_id}')
if webhook_to_post: # TODO
json_request = {"trackerId": term_uuid, Tracker.add_tracked_item(tracker_uuid, item_id)
"itemId": item_id,
"itemURL": self.full_item_url + item_id, # Tags
"term": term, for tag in tracker.get_tags():
"itemSource": item_source, msg = f'{tag};{item_id}'
"itemDate": item_date, self.send_message_to_queue(msg, 'Tags')
"tags": tags_to_add,
"emailNotification": f'{mail_to_notify}', # Mail
"trackerType": term_type if tracker.mail_export():
} # TODO add matches + custom subjects
try: self.exporters['mail'].export(tracker, item)
response = requests.post(webhook_to_post, json=json_request)
if response.status_code >= 400: # Webhook
self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: {response.reason}") if tracker.webhook_export():
except: self.exporters['webhook'].export(tracker, item)
self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: Something went wrong")
if __name__ == '__main__': if __name__ == '__main__':

2
bin/trackers/Tracker_Typo_Squatting.py
View File

@ -26,8 +26,6 @@ from exporter.MailExporter import MailExporterTracker
from exporter.WebHookExporter import WebHookExporterTracker from exporter.WebHookExporter import WebHookExporterTracker
class Tracker_Typo_Squatting(AbstractModule): class Tracker_Typo_Squatting(AbstractModule):
mail_body_template = "AIL Framework,\nNew occurrence for tracked Typo: {}\nitem id: {}\nurl: {}{}"
""" """
Tracker_Typo_Squatting module for AIL framework Tracker_Typo_Squatting module for AIL framework
""" """

70
bin/trackers/Tracker_Yara.py
View File

@ -1,8 +1,10 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
################################## """
# The Tracker_Yara trackers module The Tracker_Yara trackers module
################################## ===================
"""
################################## ##################################
# Import External packages # Import External packages
@ -11,23 +13,20 @@ import os
import sys import sys
import time import time
import yara import yara
import requests
sys.path.append(os.environ['AIL_BIN']) sys.path.append(os.environ['AIL_BIN'])
################################## ##################################
# Import Project packages # Import Project packages
################################## ##################################
from modules.abstract_module import AbstractModule from modules.abstract_module import AbstractModule
from packages import Term
from lib.objects.Items import Item from lib.objects.Items import Item
from lib import Tracker from lib import Tracker
import NotificationHelper # # TODO: refactor from exporter.MailExporter import MailExporterTracker
from exporter.WebHookExporter import WebHookExporterTracker
class Tracker_Yara(AbstractModule): class Tracker_Yara(AbstractModule):
mail_body_template = "AIL Framework,\nNew YARA match: {}\nitem id: {}\nurl: {}{}"
""" """
Tracker_Yara module for AIL framework Tracker_Yara module for AIL framework
""" """
@ -35,14 +34,16 @@ class Tracker_Yara(AbstractModule):
super(Tracker_Yara, self).__init__() super(Tracker_Yara, self).__init__()
self.pending_seconds = 5 self.pending_seconds = 5
self.full_item_url = self.process.config.get("Notifications", "ail_domain") + "/object/item?id="
# Load Yara rules # Load Yara rules
self.rules = Tracker.reload_yara_rules() self.rules = Tracker.reload_yara_rules()
self.last_refresh = time.time() self.last_refresh = time.time()
self.item = None self.item = None
# Exporter
self.exporters = {'mail': MailExporterTracker(),
'webhook': WebHookExporterTracker()}
self.redis_logger.info(f"Module: {self.module_name} Launched") self.redis_logger.info(f"Module: {self.module_name} Launched")
def compute(self, item_id, item_content=None): def compute(self, item_id, item_content=None):
@ -63,59 +64,40 @@ class Tracker_Yara(AbstractModule):
if yara_match: if yara_match:
self.redis_logger.warning(f'tracker yara: new match {self.item.get_id()}: {yara_match}') self.redis_logger.warning(f'tracker yara: new match {self.item.get_id()}: {yara_match}')
print(f'{self.item.get_id()}: {yara_match}') print(f'{self.item.get_id()}: {yara_match}')
except yara.TimeoutError as e: except yara.TimeoutError:
print(f'{self.item.get_id()}: yara scanning timed out') print(f'{self.item.get_id()}: yara scanning timed out')
self.redis_logger.info(f'{self.item.get_id()}: yara scanning timed out') self.redis_logger.info(f'{self.item.get_id()}: yara scanning timed out')
def yara_rules_match(self, data): def yara_rules_match(self, data):
tracker_uuid = data['namespace'] tracker_uuid = data['namespace']
item_id = self.item.get_id() item_id = self.item.get_id()
item = Item(item_id)
item_source = self.item.get_source() item_source = self.item.get_source()
item_date = self.item.get_date()
tracker = Tracker.Tracker(tracker_uuid)
# Source Filtering # Source Filtering
tracker_sources = Tracker.get_tracker_uuid_sources(tracker_uuid) tracker_sources = tracker.get_sources()
if tracker_sources and item_source not in tracker_sources: if tracker_sources and item_source not in tracker_sources:
print(f'Source Filtering: {data["rule"]}') print(f'Source Filtering: {data["rule"]}')
return yara.CALLBACK_CONTINUE return yara.CALLBACK_CONTINUE
Tracker.add_tracked_item(tracker_uuid, item_id) Tracker.add_tracked_item(tracker_uuid, item_id) # TODO
# Tags # Tags
tags_to_add = Tracker.get_tracker_tags(tracker_uuid) for tag in tracker.get_tags():
for tag in tags_to_add: msg = f'{tag};{item_id}'
msg = '{};{}'.format(tag, item_id)
self.send_message_to_queue(msg, 'Tags') self.send_message_to_queue(msg, 'Tags')
# Mails # Mails
mail_to_notify = Tracker.get_tracker_mails(tracker_uuid) if tracker.mail_export():
if mail_to_notify: # TODO add matches + custom subjects
mail_subject = Tracker.get_email_subject(tracker_uuid) self.exporters['mail'].export(tracker, item)
mail_body = Tracker_Yara.mail_body_template.format(data['rule'], item_id, self.full_item_url, item_id)
for mail in mail_to_notify:
self.redis_logger.debug(f'Send Mail {mail_subject}')
print(f'Send Mail {mail_subject}')
NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
# Webhook # Webhook
webhook_to_post = Term.get_term_webhook(tracker_uuid) if tracker.webhook_export():
if webhook_to_post: self.exporters['webhook'].export(tracker, item)
json_request = {"trackerId": tracker_uuid,
"itemId": item_id,
"itemURL": self.full_item_url + item_id,
"dataRule": data["rule"],
"itemSource": item_source,
"itemDate": item_date,
"tags": tags_to_add,
"emailNotification": f'{mail_to_notify}',
"trackerType": "yara"
}
try:
response = requests.post(webhook_to_post, json=json_request)
if response.status_code >= 400:
self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: {response.reason}")
except:
self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: Something went wrong")
return yara.CALLBACK_CONTINUE return yara.CALLBACK_CONTINUE