mirror of https://github.com/CIRCL/AIL-framework
chg: [Tracker_Yara module] create module class
Terrtia
parent
ed37232a2d
commit
5acb583701
114
bin/LAUNCH.sh
114
bin/LAUNCH.sh
|
|
@ -147,7 +147,9 @@ function launching_scripts {
|
|||
sleep 0.1
|
||||
echo -e $GREEN"\t* Launching scripts"$DEFAULT
|
||||
|
||||
# LAUNCH CORE MODULE
|
||||
##################################
|
||||
# CORE MODULES #
|
||||
##################################
|
||||
screen -S "Script_AIL" -X screen -t "JSON_importer" bash -c "cd ${AIL_BIN}/import; ${ENV_PY} ./JSON_importer.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Crawler_manager" bash -c "cd ${AIL_BIN}/core; ${ENV_PY} ./Crawler_manager.py; read x"
|
||||
|
|
@ -157,57 +159,77 @@ function launching_scripts {
|
|||
screen -S "Script_AIL" -X screen -t "DbCleaner" bash -c "cd ${AIL_BIN}/core; ${ENV_PY} ./DbCleaner.py; read x"
|
||||
sleep 0.1
|
||||
|
||||
##################################
|
||||
# MODULES #
|
||||
##################################
|
||||
screen -S "Script_AIL" -X screen -t "Global" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Global.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Categ" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Categ.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Indexer" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Indexer.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Tags" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Tags.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./submit_paste.py; read x"
|
||||
sleep 0.1
|
||||
|
||||
screen -S "Script_AIL" -X screen -t "ApiKey" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./ApiKey.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Credential" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Credential.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "CreditCards" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./CreditCards.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Decoder" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Decoder.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./DomClassifier.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Keys" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Keys.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Onion" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Onion.py; read x"
|
||||
sleep 0.1
|
||||
|
||||
##################################
|
||||
# TRACKERS MODULES #
|
||||
##################################
|
||||
screen -S "Script_AIL" -X screen -t "Tracker_Term" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Term.py; read x"
|
||||
sleep 0.1
|
||||
|
||||
screen -S "Script_AIL" -X screen -t "Tracker_Yara" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Yara.py; read x"
|
||||
sleep 0.1
|
||||
|
||||
##################################
|
||||
# DISABLED MODULES #
|
||||
##################################
|
||||
#screen -S "Script_AIL" -X screen -t "Phone" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Phone.py; read x"
|
||||
#sleep 0.1
|
||||
|
||||
##################################
|
||||
# #
|
||||
##################################
|
||||
screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Mixer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mixer.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Global" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Global.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Duplicates" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Duplicates.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./DomClassifier.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Categ" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Categ.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "CreditCards" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./CreditCards.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "BankAccount" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./BankAccount.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Onion" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Onion.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Mail" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mail.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "ApiKey" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ApiKey.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Urls" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Urls.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Credential" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Credential.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "TermTrackerMod" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./TermTrackerMod.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "RegexTracker" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./RegexTracker.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Tracker_Yara" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Yara.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Indexer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Indexer.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Keys" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Keys.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "PgpDump" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./PgpDump.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Decoder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Decoder.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Cryptocurrency" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Cryptocurrencies.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Telegram" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Telegram.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Tools" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tools.py; read x"
|
||||
sleep 0.1
|
||||
#screen -S "Script_AIL" -X screen -t "Phone" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Phone.py; read x"
|
||||
#sleep 0.1
|
||||
#screen -S "Script_AIL" -X screen -t "Release" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Release.py; read x"
|
||||
#sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Cve" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Cve.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModuleStats.py; read x"
|
||||
|
|
@ -218,48 +240,18 @@ function launching_scripts {
|
|||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Tags" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tags.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Languages" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Languages.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./SentimentAnalysis.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "UpdateBackground" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./update-background.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./submit_paste.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "IPAddress" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./IPAddress.py; read x"
|
||||
|
||||
}
|
||||
#screen -S "Script_AIL" -X screen -t "Release" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Release.py; read x"
|
||||
#sleep 0.1
|
||||
|
||||
# function launching_crawler {
|
||||
# if [[ ! $iscrawler ]]; then
|
||||
# CONFIG=$AIL_HOME/configs/core.cfg
|
||||
# lport=$(awk '/^\[Crawler\]/{f=1} f==1&&/^splash_port/{print $3;exit}' "${CONFIG}")
|
||||
#
|
||||
# IFS='-' read -ra PORTS <<< "$lport"
|
||||
# if [ ${#PORTS[@]} -eq 1 ]
|
||||
# then
|
||||
# first_port=${PORTS[0]}
|
||||
# last_port=${PORTS[0]}
|
||||
# else
|
||||
# first_port=${PORTS[0]}
|
||||
# last_port=${PORTS[1]}
|
||||
# fi
|
||||
#
|
||||
# screen -dmS "Crawler_AIL"
|
||||
# sleep 0.1
|
||||
#
|
||||
# for ((i=first_port;i<=last_port;i++)); do
|
||||
# screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Crawler.py $i; read x"
|
||||
# sleep 0.1
|
||||
# done
|
||||
#
|
||||
# echo -e $GREEN"\t* Launching Crawler_AIL scripts"$DEFAULT
|
||||
# else
|
||||
# echo -e $RED"\t* A screen is already launched"$DEFAULT
|
||||
# fi
|
||||
# }
|
||||
}
|
||||
|
||||
function shutting_down_redis {
|
||||
redis_dir=${AIL_HOME}/redis/src/
|
||||
|
|
@ -491,7 +483,7 @@ function update_thirdparty {
|
|||
function launch_tests() {
|
||||
tests_dir=${AIL_HOME}/tests
|
||||
bin_dir=${AIL_BIN}
|
||||
python3 `which nosetests` -w $tests_dir --with-coverage --cover-package=$bin_dir -d #--cover-erase
|
||||
python3 `which nosetests` -w $tests_dir --with-coverage --cover-package=$bin_dir -d --cover-erase
|
||||
}
|
||||
|
||||
function reset_password() {
|
||||
|
|
|
|||
|
|
@ -7,27 +7,27 @@ The Phone Module
|
|||
|
||||
This module is consuming the Redis-list created by the Categ module.
|
||||
|
||||
It apply phone number regexes on paste content and warn if above a threshold.
|
||||
It apply phone number regexes on item content and warn if above a threshold.
|
||||
|
||||
"""
|
||||
|
||||
##################################
|
||||
# Import External packages
|
||||
##################################
|
||||
import time
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import phonenumbers
|
||||
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from module.abstract_module import AbstractModule
|
||||
from packages import Paste
|
||||
from pubsublogger import publisher
|
||||
from Helper import Process
|
||||
|
||||
from modules.abstract_module import AbstractModule
|
||||
from packages.Item import Item
|
||||
|
||||
# # TODO: # FIXME: improve regex / filter false positives
|
||||
class Phone(AbstractModule):
|
||||
"""
|
||||
Phone module for AIL framework
|
||||
|
|
@ -46,21 +46,21 @@ class Phone(AbstractModule):
|
|||
|
||||
|
||||
def compute(self, message):
|
||||
paste = Paste.Paste(message)
|
||||
content = paste.get_p_content()
|
||||
# List of the regex results in the Paste, may be null
|
||||
item = Item(message)
|
||||
content = item.get_content()
|
||||
# List of the regex results in the Item, may be null
|
||||
results = self.REG_PHONE.findall(content)
|
||||
|
||||
# If the list is greater than 4, we consider the Paste may contain a list of phone numbers
|
||||
# If the list is greater than 4, we consider the Item may contain a list of phone numbers
|
||||
if len(results) > 4:
|
||||
self.redis_logger.debug(results)
|
||||
self.redis_logger.warning(f'{paste.p_name} contains PID (phone numbers)')
|
||||
self.redis_logger.warning(f'{item.get_id()} contains PID (phone numbers)')
|
||||
|
||||
msg = f'infoleak:automatic-detection="phone-number";{message}'
|
||||
self.process.populate_set_out(msg, 'Tags')
|
||||
msg = f'infoleak:automatic-detection="phone-number";{item.get_id()}'
|
||||
self.send_message_to_queue(msg, 'Tags')
|
||||
|
||||
# Send to duplicate
|
||||
self.process.populate_set_out(message, 'Duplicate')
|
||||
self.send_message_to_queue(item.get_id(), 'Duplicate')
|
||||
|
||||
stats = {}
|
||||
for phone_number in results:
|
||||
|
|
@ -75,10 +75,10 @@ class Phone(AbstractModule):
|
|||
pass
|
||||
for country_code in stats:
|
||||
if stats[country_code] > 4:
|
||||
self.redis_logger.warning(f'{paste.p_name} contains Phone numbers with country code {country_code}')
|
||||
self.redis_logger.warning(f'{item.get_id()} contains Phone numbers with country code {country_code}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
|
||||
module = Phone()
|
||||
module.run()
|
||||
|
|
|
|||
|
|
@ -26,16 +26,14 @@ import calendar
|
|||
import redis
|
||||
import json
|
||||
import signal
|
||||
from pubsublogger import publisher
|
||||
from nltk.sentiment.vader import SentimentIntensityAnalyzer
|
||||
from nltk import tokenize, download
|
||||
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from module.abstract_module import AbstractModule
|
||||
from Helper import Process
|
||||
from modules.abstract_module import AbstractModule
|
||||
from packages import Paste
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
|
|
@ -49,13 +47,13 @@ def timeout_handler(signum, frame):
|
|||
|
||||
signal.signal(signal.SIGALRM, timeout_handler)
|
||||
|
||||
|
||||
## TODO: REFACTOR MODULE + CLEAN HISTORY
|
||||
class SentimentAnalysis(AbstractModule):
|
||||
"""
|
||||
SentimentAnalysis module for AIL framework
|
||||
"""
|
||||
|
||||
|
||||
|
||||
# Config Variables
|
||||
accepted_Mime_type = ['text/plain']
|
||||
line_max_length_threshold = 1000
|
||||
|
|
|
|||
|
|
@ -12,11 +12,14 @@ This module add tags to an item.
|
|||
##################################
|
||||
# Import External packages
|
||||
##################################
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from module.abstract_module import AbstractModule
|
||||
from modules.abstract_module import AbstractModule
|
||||
from packages.Item import Item
|
||||
from packages import Tag
|
||||
|
||||
|
|
|
|||
|
|
@ -23,21 +23,18 @@ import time
|
|||
# from sflock.main import unpack
|
||||
# import sflock
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from module.abstract_module import AbstractModule
|
||||
from Helper import Process
|
||||
from pubsublogger import publisher
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
||||
import Tag
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
from modules.abstract_module import AbstractModule
|
||||
from packages import Tag
|
||||
from lib import ConfigLoader
|
||||
|
||||
|
||||
class SubmitPaste(AbstractModule):
|
||||
"""
|
||||
Company Credentials module for AIL framework
|
||||
SubmitPaste module for AIL framework
|
||||
"""
|
||||
|
||||
expire_time = 120
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ import Tracker
|
|||
|
||||
from flask import escape
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
||||
import Date
|
||||
import Item
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ publish = Redis_D4_client
|
|||
[D4_client]
|
||||
subscribe = Redis_D4_client
|
||||
|
||||
[TermTrackerMod]
|
||||
[Tracker_Term]
|
||||
subscribe = Redis_Global
|
||||
publish = Redis_Tags
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
The TermTracker Module
|
||||
The Tracker_Term Module
|
||||
===================
|
||||
|
||||
"""
|
||||
|
|
@ -14,12 +14,11 @@ import sys
|
|||
import time
|
||||
import signal
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from Helper import Process
|
||||
from pubsublogger import publisher
|
||||
from module.abstract_module import AbstractModule
|
||||
from modules.abstract_module import AbstractModule
|
||||
import NotificationHelper
|
||||
from packages import Item
|
||||
from packages import Term
|
||||
|
|
@ -33,19 +32,19 @@ def timeout_handler(signum, frame):
|
|||
signal.signal(signal.SIGALRM, timeout_handler)
|
||||
|
||||
|
||||
class TermTrackerMod(AbstractModule):
|
||||
class Tracker_Term(AbstractModule):
|
||||
|
||||
mail_body_template = "AIL Framework,\nNew occurrence for term tracked term: {}\nitem id: {}\nurl: {}{}"
|
||||
mail_body_template = "AIL Framework,\nNew occurrence for tracked term: {}\nitem id: {}\nurl: {}{}"
|
||||
|
||||
"""
|
||||
TermTrackerMod module for AIL framework
|
||||
Tracker_Term module for AIL framework
|
||||
"""
|
||||
def __init__(self):
|
||||
super(TermTrackerMod, self).__init__()
|
||||
super(Tracker_Term, self).__init__()
|
||||
|
||||
self.pending_seconds = 5
|
||||
|
||||
self.max_execution_time = self.process.config.getint('TermTrackerMod', "max_execution_time")
|
||||
self.max_execution_time = self.process.config.getint('Tracker_Term', "max_execution_time")
|
||||
|
||||
self.full_item_url = self.process.config.get("Notifications", "ail_domain") + "/object/item?id="
|
||||
|
||||
|
|
@ -55,8 +54,7 @@ class TermTrackerMod(AbstractModule):
|
|||
self.set_tracked_words_list = Term.get_set_tracked_words_list()
|
||||
self.last_refresh_set = time.time()
|
||||
|
||||
# Send module state to logs
|
||||
self.redis_logger.info("Module %s initialized"%(self._module_name()))
|
||||
self.redis_logger.info(f"Module: {self.module_name} Launched")
|
||||
|
||||
|
||||
def compute(self, item_id):
|
||||
|
|
@ -72,8 +70,9 @@ class TermTrackerMod(AbstractModule):
|
|||
self.redis_logger.debug('Tracked set refreshed')
|
||||
|
||||
# Cast message as Item
|
||||
item_date = Item.get_item_date(item_id)
|
||||
item_content = Item.get_item_content(item_id)
|
||||
item = Item(item_id)
|
||||
item_date = item.get_date()
|
||||
item_content = item.get_content()
|
||||
|
||||
signal.alarm(self.max_execution_time)
|
||||
|
||||
|
|
@ -81,7 +80,7 @@ class TermTrackerMod(AbstractModule):
|
|||
try:
|
||||
dict_words_freq = Term.get_text_word_frequency(item_content)
|
||||
except TimeoutException:
|
||||
self.redis_logger.warning("{0} processing timeout".format(item_id))
|
||||
self.redis_logger.warning(f"{item.get_id()} processing timeout")
|
||||
else:
|
||||
signal.alarm(0)
|
||||
|
||||
|
|
@ -93,7 +92,7 @@ class TermTrackerMod(AbstractModule):
|
|||
# check solo words
|
||||
for word in self.list_tracked_words:
|
||||
if word in dict_words_freq:
|
||||
self.new_term_found(word, 'word', item_id, item_date)
|
||||
self.new_term_found(word, 'word', item.get_id(), item_date)
|
||||
|
||||
# check words set
|
||||
for elem in self.set_tracked_words_list:
|
||||
|
|
@ -106,11 +105,12 @@ class TermTrackerMod(AbstractModule):
|
|||
if word in dict_words_freq:
|
||||
nb_uniq_word += 1
|
||||
if nb_uniq_word >= nb_words_threshold:
|
||||
self.new_term_found(word_set, 'set', item_id, item_date)
|
||||
self.new_term_found(word_set, 'set', item.get_id(), item_date)
|
||||
|
||||
def new_term_found(self, term, term_type, item_id, item_date):
|
||||
uuid_list = Term.get_term_uuid_list(term, term_type)
|
||||
self.redis_logger.info('new tracked term found: {} in {}'.format(term, item_id))
|
||||
self.redis_logger.info(f'new tracked term found: {term} in {item_id}')
|
||||
print(f'new tracked term found: {term} in {item_id}')
|
||||
|
||||
for term_uuid in uuid_list:
|
||||
Term.add_tracked_item(term_uuid, item_id, item_date)
|
||||
|
|
@ -118,18 +118,19 @@ class TermTrackerMod(AbstractModule):
|
|||
tags_to_add = Term.get_term_tags(term_uuid)
|
||||
for tag in tags_to_add:
|
||||
msg = '{};{}'.format(tag, item_id)
|
||||
self.process.populate_set_out(msg, 'Tags')
|
||||
self.send_message_to_queue(msg, 'Tags')
|
||||
|
||||
mail_to_notify = Term.get_term_mails(term_uuid)
|
||||
if mail_to_notify:
|
||||
mail_subject = Tracker.get_email_subject(term_uuid)
|
||||
mail_body = TermTrackerMod.mail_body_template.format(term, item_id, self.full_item_url, item_id)
|
||||
mail_body = Tracker_Term.mail_body_template.format(term, item_id, self.full_item_url, item_id)
|
||||
for mail in mail_to_notify:
|
||||
self.redis_logger.debug('Send Mail {}'.format(mail_subject))
|
||||
self.redis_logger.debug(f'Send Mail {mail_subject}')
|
||||
print(f'Send Mail {mail_subject}')
|
||||
NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
module = TermTrackerMod()
|
||||
module = Tracker_Term()
|
||||
module.run()
|
||||
|
|
|
|||
|
|
@ -1,87 +1,99 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
Yara trackers
|
||||
The Tracker_Yara trackers module
|
||||
===================
|
||||
|
||||
"""
|
||||
|
||||
##################################
|
||||
# Import External packages
|
||||
##################################
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import yara
|
||||
|
||||
from pubsublogger import publisher
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from Helper import Process
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from modules.abstract_module import AbstractModule
|
||||
from packages import Term
|
||||
from packages.Item import Item
|
||||
from lib import Tracker
|
||||
|
||||
import NotificationHelper # # TODO: refractor
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
|
||||
import Term
|
||||
class Tracker_Yara(AbstractModule):
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
|
||||
import Tracker
|
||||
import item_basic
|
||||
mail_body_template = "AIL Framework,\nNew YARA match: {}\nitem id: {}\nurl: {}{}"
|
||||
|
||||
"""
|
||||
Tracker_Yara module for AIL framework
|
||||
"""
|
||||
def __init__(self):
|
||||
super(Tracker_Yara, self).__init__()
|
||||
self.pending_seconds = 5
|
||||
|
||||
self.full_item_url = self.process.config.get("Notifications", "ail_domain") + "/object/item?id="
|
||||
|
||||
# Load Yara rules
|
||||
self.rules = Tracker.reload_yara_rules()
|
||||
self.last_refresh = time.time()
|
||||
|
||||
self.item = None
|
||||
|
||||
self.redis_logger.info(f"Module: {self.module_name} Launched")
|
||||
|
||||
|
||||
full_item_url = "/object/item?id="
|
||||
mail_body_template = "AIL Framework,\nNew YARA match: {}\nitem id: {}\nurl: {}{}"
|
||||
|
||||
last_refresh = time.time()
|
||||
|
||||
def yara_rules_match(data):
|
||||
#print(data)
|
||||
tracker_uuid = data['namespace']
|
||||
|
||||
item_date = item_basic.get_item_date(item_id)
|
||||
Tracker.add_tracked_item(tracker_uuid, item_id, item_date)
|
||||
|
||||
# Tags
|
||||
tags_to_add = Tracker.get_tracker_tags(tracker_uuid)
|
||||
for tag in tags_to_add:
|
||||
msg = '{};{}'.format(tag, item_id)
|
||||
p.populate_set_out(msg, 'Tags')
|
||||
|
||||
# Mails
|
||||
mail_to_notify = Tracker.get_tracker_mails(tracker_uuid)
|
||||
if mail_to_notify:
|
||||
mail_subject = Tracker.get_email_subject(tracker_uuid)
|
||||
mail_body = mail_body_template.format(data['rule'], item_id, full_item_url, item_id)
|
||||
for mail in mail_to_notify:
|
||||
NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
|
||||
|
||||
return yara.CALLBACK_CONTINUE
|
||||
|
||||
if __name__ == "__main__":
|
||||
publisher.port = 6380
|
||||
publisher.channel = "Script"
|
||||
publisher.info("Script Tracker_Yara started")
|
||||
|
||||
config_section = 'Tracker_Yara'
|
||||
module_name = "Tracker_Yara"
|
||||
p = Process(config_section)
|
||||
|
||||
full_item_url = p.config.get("Notifications", "ail_domain") + full_item_url
|
||||
|
||||
# Load Yara rules
|
||||
rules = Tracker.reload_yara_rules()
|
||||
|
||||
# Regex Frequency
|
||||
while True:
|
||||
item_id = p.get_from_set()
|
||||
if item_id is not None:
|
||||
item_content = item_basic.get_item_content(item_id)
|
||||
try:
|
||||
yara_match = rules.match(data=item_content, callback=yara_rules_match, which_callbacks=yara.CALLBACK_MATCHES, timeout=60)
|
||||
if yara_match:
|
||||
print(f'{item_id}: {yara_match}')
|
||||
except yara.TimeoutError as e:
|
||||
print(f'{item_id}: yara scanning timed out')
|
||||
else:
|
||||
time.sleep(5)
|
||||
|
||||
def compute(self, item_id):
|
||||
# refresh YARA list
|
||||
if last_refresh < Tracker.get_tracker_last_updated_by_type('yara'):
|
||||
rules = Tracker.reload_yara_rules()
|
||||
last_refresh = time.time()
|
||||
if self.last_refresh < Tracker.get_tracker_last_updated_by_type('yara'):
|
||||
self.rules = Tracker.reload_yara_rules()
|
||||
self.last_refresh = time.time()
|
||||
self.redis_logger.debug('Tracked set refreshed')
|
||||
print('Tracked set refreshed')
|
||||
|
||||
self.item = Item(item_id)
|
||||
item_content = self.item.get_content()
|
||||
try:
|
||||
yara_match = self.rules.match(data=item_content, callback=self.yara_rules_match, which_callbacks=yara.CALLBACK_MATCHES, timeout=60)
|
||||
if yara_match:
|
||||
self.redis_logger.info(f'{self.item.get_id()}: {yara_match}')
|
||||
print(f'{self.item.get_id()}: {yara_match}')
|
||||
except yara.TimeoutError as e:
|
||||
print(f'{self.item.get_id()}: yara scanning timed out')
|
||||
self.redis_logger.info(f'{self.item.get_id()}: yara scanning timed out')
|
||||
|
||||
def yara_rules_match(self, data):
|
||||
tracker_uuid = data['namespace']
|
||||
|
||||
item_id = self.item.get_id()
|
||||
item_date = self.item.get_date()
|
||||
Tracker.add_tracked_item(tracker_uuid, item_id, item_date)
|
||||
|
||||
# Tags
|
||||
tags_to_add = Tracker.get_tracker_tags(tracker_uuid)
|
||||
for tag in tags_to_add:
|
||||
msg = '{};{}'.format(tag, item_id)
|
||||
self.send_message_to_queue(msg, 'Tags')
|
||||
|
||||
# Mails
|
||||
mail_to_notify = Tracker.get_tracker_mails(tracker_uuid)
|
||||
if mail_to_notify:
|
||||
mail_subject = Tracker.get_email_subject(tracker_uuid)
|
||||
mail_body = Tracker_Yara.mail_body_template.format(data['rule'], item_id, self.full_item_url, item_id)
|
||||
for mail in mail_to_notify:
|
||||
self.redis_logger.debug(f'Send Mail {mail_subject}')
|
||||
print(f'Send Mail {mail_subject}')
|
||||
NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
|
||||
|
||||
return yara.CALLBACK_CONTINUE
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
module = Tracker_Yara()
|
||||
module.run()
|
||||
|
|
|
|||
|
|
@ -116,7 +116,7 @@ operation_mode = 3
|
|||
ttl_duplicate = 86400
|
||||
default_unnamed_feed_name = unnamed_feeder
|
||||
|
||||
[TermTrackerMod]
|
||||
[Tracker_Term]
|
||||
max_execution_time = 120
|
||||
|
||||
[RegexTracker]
|
||||
|
|
@ -253,10 +253,6 @@ address = tcp://127.0.0.1:5556,tcp://crf.circl.lu:5556
|
|||
channel = 102
|
||||
bind = tcp://127.0.0.1:5556
|
||||
|
||||
[ZMQ_Url]
|
||||
address = tcp://127.0.0.1:5004
|
||||
channel = urls
|
||||
|
||||
[ZMQ_FetchedOnion]
|
||||
address = tcp://127.0.0.1:5005
|
||||
channel = FetchedOnion
|
||||
|
|
|
|||
Loading…
Reference in New Issue