CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [pubsublogger] remove old redis_logger

dashboard
terrtia 2024-12-09 13:54:54 +01:00
parent feabe9f5df
commit 849977f0c2
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
32 changed files with 112 additions and 241 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
bin/LAUNCH.sh
View File

@ -615,14 +615,13 @@ function launch_all {
update;
launch_redis;
launch_kvrocks;
launch_logs;
launch_scripts;
launch_flask;
}
function menu_display {
options=("Redis" "Kvrocks" "Logs" "Scripts" "Flask" "Killall" "Update" "Update-config" "Update-thirdparty")
options=("Redis" "Kvrocks" "Scripts" "Flask" "Killall" "Update" "Update-config" "Update-thirdparty")
menu() {
echo "What do you want to Launch?:"
@ -653,9 +652,6 @@ function menu_display {
Kvrocks)
launch_kvrocks;
;;
Logs)
launch_logs;
;;
Scripts)
launch_scripts;
;;

1
bin/core/Sync_module.py
View File

@ -88,7 +88,6 @@ class Sync_module(AbstractModule):
if self.last_refresh_queues < time.time():
timeout_processed_objs()
self.last_refresh_queues = time.time() + 120
self.redis_logger.debug('Timeout queues')
# print('Timeout queues')
# Get one message (paste) from the QueueIn (copy of Redis_Global publish)

16
bin/modules/ApiKey.py
View File

@ -20,7 +20,6 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages
##################################
from modules.abstract_module import AbstractModule
from lib.objects.Items import Item
class ApiKey(AbstractModule):
"""ApiKey module for AIL framework"""
@ -48,20 +47,19 @@ class ApiKey(AbstractModule):
def compute(self, message, r_result=False):
score = message
item = self.get_obj()
item_content = item.get_content()
obj = self.get_obj()
content = obj.get_content()
google_api_key = self.regex_findall(self.re_google_api_key, item.get_id(), item_content, r_set=True)
aws_access_key = self.regex_findall(self.re_aws_access_key, item.get_id(), item_content, r_set=True)
google_api_key = self.regex_findall(self.re_google_api_key, obj.get_id(), content, r_set=True)
aws_access_key = self.regex_findall(self.re_aws_access_key, obj.get_id(), content, r_set=True)
if aws_access_key:
aws_secret_key = self.regex_findall(self.re_aws_secret_key, item.get_id(), item_content, r_set=True)
aws_secret_key = self.regex_findall(self.re_aws_secret_key, obj.get_id(), content, r_set=True)
if aws_access_key or google_api_key:
to_print = f'ApiKey;{item.get_source()};{item.get_date()};{item.get_basename()};'
to_print = obj.get_global_id()
if google_api_key:
print(f'found google api key: {to_print}')
self.redis_logger.warning(f'{to_print}Checked {len(google_api_key)} found Google API Key;{self.obj.get_global_id()}')
tag = 'infoleak:automatic-detection="google-api-key"'
self.add_message_to_queue(message=tag, queue='Tags')
@ -69,10 +67,8 @@ class ApiKey(AbstractModule):
# # TODO: # FIXME: AWS regex/validate/sanitize KEY + SECRET KEY
if aws_access_key:
print(f'found AWS key: {to_print}')
self.redis_logger.warning(f'{to_print}Checked {len(aws_access_key)} found AWS Key;{self.obj.get_global_id()}')
if aws_secret_key:
print(f'found AWS secret key')
self.redis_logger.warning(f'{to_print}Checked {len(aws_secret_key)} found AWS secret Key;{self.obj.get_global_id()}')
tag = 'infoleak:automatic-detection="aws-key"'
self.add_message_to_queue(message=tag, queue='Tags')

5
bin/modules/Categ.py
View File

@ -65,7 +65,7 @@ class Categ(AbstractModule):
self.matchingThreshold = config_loader.get_config_int("Categ", "matchingThreshold")
self.reload_categ_words()
self.redis_logger.info("Script Categ started")
self.logger.info("Script Categ started")
# # TODO: trigger reload on change ( save last reload time, ...)
def reload_categ_words(self):
@ -103,9 +103,6 @@ class Categ(AbstractModule):
print(msg, categ)
self.add_message_to_queue(message=msg, queue=categ)
self.redis_logger.debug(
f'Categ;{obj.get_source()};{obj.get_date()};{obj.get_basename()};Detected {lenfound} as {categ};{obj.get_id()}')
if r_result:
return categ_found

22
bin/modules/Credential.py
View File

@ -29,7 +29,6 @@ Redis organization:
import os
import sys
import time
from datetime import datetime
from pyfaup.faup import Faup
sys.path.append(os.environ['AIL_BIN'])
@ -37,9 +36,7 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages
##################################
from modules.abstract_module import AbstractModule
from lib.objects.Items import Item
from lib import ConfigLoader
# from lib import Statistics
class Credential(AbstractModule):
@ -80,40 +77,36 @@ class Credential(AbstractModule):
self.pending_seconds = 10
# Send module state to logs
self.redis_logger.info(f"Module {self.module_name} initialized")
self.logger.info(f"Module {self.module_name} initialized")
def compute(self, message):
count = message
item = self.get_obj()
obj = self.get_obj()
item_content = item.get_content()
content = obj.get_content()
# TODO: USE SETS
# Extract all credentials
all_credentials = self.regex_findall(self.regex_cred, item.get_id(), item_content)
all_credentials = self.regex_findall(self.regex_cred, obj.get_id(), content)
if all_credentials:
nb_cred = len(all_credentials)
message = f'Checked {nb_cred} credentials found.'
all_sites = self.regex_findall(self.regex_web, item.get_id(), item_content, r_set=True)
all_sites = self.regex_findall(self.regex_web, obj.get_id(), content, r_set=True)
if all_sites:
discovered_sites = ', '.join(all_sites)
message += f' Related websites: {discovered_sites}'
print(message)
to_print = f'Credential;{item.get_source()};{item.get_date()};{item.get_basename()};{message};{self.obj.get_global_id()}'
# num of creds above threshold, publish an alert
if nb_cred > self.criticalNumberToAlert:
print(f"========> Found more than 10 credentials in this file : {self.obj.get_global_id()}")
self.redis_logger.warning(to_print)
tag = 'infoleak:automatic-detection="credential"'
self.add_message_to_queue(message=tag, queue='Tags')
site_occurrence = self.regex_findall(self.regex_site_for_stats, item.get_id(), item_content)
site_occurrence = self.regex_findall(self.regex_site_for_stats, obj.get_id(), content)
creds_sites = {}
@ -162,8 +155,7 @@ class Credential(AbstractModule):
# for tld in nb_tlds:
# Statistics.add_module_tld_stats_by_date('credential', date, tld, nb_tlds[tld])
else:
self.redis_logger.info(to_print)
print(f'found {nb_cred} credentials')
print(f'found {nb_cred} credentials {self.obj.get_global_id()}')
# # TODO: # FIXME: TEMP DESABLE
# # For searching credential in termFreq

17
bin/modules/CreditCards.py
View File

@ -7,7 +7,7 @@ The CreditCards Module
This module is consuming the Redis-list created by the Categ module.
It apply credit card regexes on item content and warn if a valid card number is found.
It apply credit card regexes on object content and warn if a valid card number is found.
"""
@ -23,7 +23,6 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages
##################################
from modules.abstract_module import AbstractModule
from lib.objects.Items import Item
from packages import lib_refine
class CreditCards(AbstractModule):
@ -68,10 +67,9 @@ class CreditCards(AbstractModule):
return extracted
def compute(self, message, r_result=False):
score = message
item = self.get_obj()
content = item.get_content()
all_cards = self.regex_findall(self.regex, item.id, content)
obj = self.get_obj()
content = obj.get_content()
all_cards = self.regex_findall(self.regex, obj.id, content)
if len(all_cards) > 0:
# self.logger.debug(f'All matching {all_cards}')
@ -84,11 +82,8 @@ class CreditCards(AbstractModule):
creditcard_set.add(valid_card)
# print(creditcard_set)
to_print = f'CreditCard;{item.get_source()};{item.get_date()};{item.get_basename()};'
if creditcard_set:
mess = f'{to_print}Checked {len(creditcard_set)} valid number(s);{self.obj.get_global_id()}'
print(mess)
self.redis_logger.warning(mess)
print(f'{len(creditcard_set)} valid number(s);{self.obj.get_global_id()}')
tag = 'infoleak:automatic-detection="credit-card"'
self.add_message_to_queue(message=tag, queue='Tags')
@ -96,7 +91,7 @@ class CreditCards(AbstractModule):
if r_result:
return creditcard_set
else:
self.redis_logger.info(f'{to_print}CreditCard related;{self.obj.get_global_id()}')
print(f'CreditCard related;{self.obj.get_global_id()}')
if __name__ == '__main__':

10
bin/modules/Cryptocurrencies.py
View File

@ -28,7 +28,6 @@ sys.path.append(os.environ['AIL_BIN'])
##################################
from modules.abstract_module import AbstractModule
from lib.objects.CryptoCurrencies import CryptoCurrency
from lib.objects.Items import Item
##################################
##################################
@ -151,17 +150,10 @@ class Cryptocurrencies(AbstractModule, ABC):
# debug
print(private_keys)
to_print = 'Cryptocurrency;{};{};{};'.format(item.get_source(),
item.get_date(),
item.get_basename())
self.redis_logger.warning('{}Detected {} {} private key;{}'.format(
to_print, len(private_keys), currency['name'], self.obj.get_global_id()))
else:
private_keys = []
to_print = f"{currency['name']} found: {len(addresses)} address and {len(private_keys)} private Keys"
print(to_print)
self.redis_logger.warning(to_print)
print(f"{currency['name']} found: {len(addresses)} address and {len(private_keys)} private Keys {self.obj.get_global_id()}")
if __name__ == '__main__':

5
bin/modules/CveModule.py
View File

@ -44,7 +44,6 @@ class CveModule(AbstractModule):
self.logger.info(f'Module {self.module_name} initialized')
def compute(self, message):
count = message
item = self.get_obj()
item_id = item.get_id()
@ -56,9 +55,7 @@ class CveModule(AbstractModule):
cve = Cves.Cve(cve_id)
cve.add(date, item)
warning = f'{self.obj.get_global_id()} contains CVEs {cves}'
print(warning)
self.redis_logger.warning(warning)
print(f'{self.obj.get_global_id()} contains CVEs {cves}')
tag = 'infoleak:automatic-detection="cve"'
# Send to Tags Queue

10
bin/modules/DomClassifier.py
View File

@ -58,10 +58,6 @@ class DomClassifier(AbstractModule):
def compute(self, message, r_result=False):
host = message
item = self.get_obj()
item_basename = item.get_basename()
item_date = item.get_date()
item_source = item.get_source()
try:
self.dom_classifier.text(rawtext=host)
@ -82,19 +78,19 @@ class DomClassifier(AbstractModule):
localizeddomains = self.dom_classifier.include(expression=self.cc_tld)
if localizeddomains:
print(localizeddomains)
self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc_tld};{self.obj.get_global_id()}")
self.logger.info(f"{localizeddomains} located in {self.cc_tld};{self.obj.get_global_id()}")
if self.cc:
localizeddomains = self.dom_classifier.localizedomain(cc=self.cc)
if localizeddomains:
print(localizeddomains)
self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc};{self.obj.get_global_id()}")
self.logger.info(f"{localizeddomains} located in {self.cc};{self.obj.get_global_id()}")
if r_result:
return self.dom_classifier.vdomain
except IOError as err:
self.redis_logger.error(f"Duplicate;{item_source};{item_date};{item_basename};CRC Checksum Failed")
self.logger.error(f"{self.obj.get_global_id()};CRC Checksum Failed")
raise Exception(f"CRC Checksum Failed on: {self.obj.get_global_id()}")

5
bin/modules/Duplicates.py
View File

@ -27,7 +27,6 @@ sys.path.append(os.environ['AIL_BIN'])
from modules.abstract_module import AbstractModule
from lib.ConfigLoader import ConfigLoader
from lib import Duplicate
from lib.objects.Items import Item
class Duplicates(AbstractModule):
@ -92,11 +91,11 @@ class Duplicates(AbstractModule):
Duplicate.save_object_hash(algo, curr_date_ymonth, self.algos[algo]['hash'], item.get_id())
if nb_duplicates:
self.redis_logger.info(f'Duplicate;{item.get_source()};{item.get_date()};{item.get_basename()};Detected {nb_duplicates};{self.obj.get_global_id()}')
self.logger.info(f'Duplicates {nb_duplicates};{self.obj.get_global_id()}')
y = time.time()
print(f'{self.obj.get_global_id()} Processed in {y-x} sec')
# self.redis_logger.debug('{}Processed in {} sec'.format(to_print, y-x))
# self.logger.debug('{}Processed in {} sec'.format(to_print, y-x))
if __name__ == "__main__":

1
bin/modules/Global.py
View File

@ -74,7 +74,6 @@ class Global(AbstractModule):
if int(difftime) > 30:
to_print = f'Global; ; ; ;glob Processed {self.processed_item} item(s) in {difftime} s'
print(to_print)
self.redis_logger.debug(to_print)
self.time_last_stats = time.time()
self.processed_item = 0

9
bin/modules/IPAddress.py
View File

@ -25,7 +25,6 @@ sys.path.append(os.environ['AIL_BIN'])
##################################
from modules.abstract_module import AbstractModule
from lib.ConfigLoader import ConfigLoader
from lib.objects.Items import Item
from lib import regex_helper
# TODO REWRITE ME -> PERF + IPV6 + Tracker ?
@ -66,11 +65,11 @@ class IPAddress(AbstractModule):
if not self.ip_networks:
return None
item = self.get_obj()
content = item.get_content()
obj = self.get_obj()
content = obj.get_content()
# list of the regex results in the Item
results = self.regex_findall(self.re_ipv4, item.get_id(), content)
results = self.regex_findall(self.re_ipv4, obj.get_id(), content)
results = set(results)
matching_ips = []
for ip in results:
@ -83,7 +82,6 @@ class IPAddress(AbstractModule):
if len(matching_ips) > 0:
self.logger.info(f'{self.obj.get_global_id()} contains {len(matching_ips)} IPs')
self.redis_logger.warning(f'{self.obj.get_global_id()} contains IPs')
# Tag message with IP
tag = 'infoleak:automatic-detection="ip"'
@ -93,4 +91,3 @@ class IPAddress(AbstractModule):
if __name__ == "__main__":
module = IPAddress()
module.run()
# module.compute('submitted/2023/05/15/submitted_8a6136c2-c7f2-4c9e-8f29-e1a62315b482.gz')

21
bin/modules/Iban.py
View File

@ -24,8 +24,7 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages
##################################
from modules.abstract_module import AbstractModule
from lib.objects.Items import Item
from lib.ConfigLoader import ConfigLoader
# from lib.ConfigLoader import ConfigLoader
# from lib import Statistics
class Iban(AbstractModule):
@ -40,7 +39,7 @@ class Iban(AbstractModule):
def __init__(self, queue=True):
super(Iban, self).__init__(queue=queue)
# Waiting time in secondes between to message proccessed
# Waiting time in seconds between to message processed
self.pending_seconds = 10
self.regex_timeout = 30
@ -49,7 +48,7 @@ class Iban(AbstractModule):
self.iban_regex_verify = re.compile(r'^([A-Z]{2})([0-9]{2})([A-Z0-9]{9,30})$')
# Send module state to logs
self.redis_logger.info(f'Module {self.module_name} initialized')
self.logger.info(f'Module {self.module_name} initialized')
def get_iban_number(self, iban):
return (iban[4:] + iban[:4]).translate(Iban.LETTERS_IBAN)
@ -73,29 +72,27 @@ class Iban(AbstractModule):
return extracted
def compute(self, message):
item = self.get_obj()
item_id = item.get_id()
obj = self.get_obj()
obj_id = obj.get_id()
ibans = self.regex_findall(self.iban_regex, item_id, item.get_content())
ibans = self.regex_findall(self.iban_regex, obj_id, obj.get_content())
if ibans:
valid_ibans = set()
for iban in ibans:
iban = iban[1:-1].replace("'", "").split(',')
iban = iban[0]+iban[1]+iban[2]
iban = ''.join(e for e in iban if e.isalnum())
if self.regex_findall(self.iban_regex_verify, item_id, iban):
if self.regex_findall(self.iban_regex_verify, obj_id, iban):
print(f'checking {iban}')
if self.is_valid_iban(iban):
valid_ibans.add(iban)
if valid_ibans:
print(f'{valid_ibans} ibans {item_id}')
date = datetime.datetime.now().strftime("%Y%m")
print(f'{valid_ibans} ibans {self.obj.get_global_id()}')
# date = datetime.datetime.now().strftime("%Y%m")
# for iban in valid_ibans:
# Statistics.add_module_tld_stats_by_date('iban', date, iban[0:2], 1)
to_print = f'Iban;{item.get_source()};{item.get_date()};{item.get_basename()};'
self.redis_logger.warning(f'{to_print}Checked found {len(valid_ibans)} IBAN;{self.obj.get_global_id()}')
# Tags
tag = 'infoleak:automatic-detection="iban"'
self.add_message_to_queue(message=tag, queue='Tags')

29
bin/modules/Keys.py
View File

@ -24,7 +24,6 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages
##################################
from modules.abstract_module import AbstractModule
from lib.objects.Items import Item
class KeyEnum(Enum):
@ -56,14 +55,14 @@ class Keys(AbstractModule):
self.pending_seconds = 1
def compute(self, message):
item = self.get_obj()
content = item.get_content()
obj = self.get_obj()
content = obj.get_content()
# find = False
get_pgp_content = False
if KeyEnum.PGP_MESSAGE.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has a PGP enc message')
print(f'{self.obj.get_global_id()} has a PGP enc message')
tag = 'infoleak:automatic-detection="pgp-message"'
self.add_message_to_queue(message=tag, queue='Tags')
@ -81,21 +80,21 @@ class Keys(AbstractModule):
get_pgp_content = True
if KeyEnum.PGP_PRIVATE_KEY_BLOCK.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has a pgp private key block message')
print(f'{self.obj.get_global_id()} has a pgp private key block message')
tag = 'infoleak:automatic-detection="pgp-private-key"'
self.add_message_to_queue(message=tag, queue='Tags')
get_pgp_content = True
if KeyEnum.CERTIFICATE.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has a certificate message')
print(f'{self.obj.get_global_id()} has a certificate message')
tag = 'infoleak:automatic-detection="certificate"'
self.add_message_to_queue(message=tag, queue='Tags')
# find = True
if KeyEnum.RSA_PRIVATE_KEY.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has a RSA private key message')
print(f'{self.obj.get_global_id()} has a RSA private key message')
print('rsa private key message found')
tag = 'infoleak:automatic-detection="rsa-private-key"'
@ -103,7 +102,7 @@ class Keys(AbstractModule):
# find = True
if KeyEnum.PRIVATE_KEY.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has a private key message')
print(f'{self.obj.get_global_id()} has a private key message')
print('private key message found')
tag = 'infoleak:automatic-detection="private-key"'
@ -111,7 +110,7 @@ class Keys(AbstractModule):
# find = True
if KeyEnum.ENCRYPTED_PRIVATE_KEY.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has an encrypted private key message')
print(f'{self.obj.get_global_id()} has an encrypted private key message')
print('encrypted private key message found')
tag = 'infoleak:automatic-detection="encrypted-private-key"'
@ -119,7 +118,7 @@ class Keys(AbstractModule):
# find = True
if KeyEnum.OPENSSH_PRIVATE_KEY.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has an openssh private key message')
print(f'{self.obj.get_global_id()} has an openssh private key message')
print('openssh private key message found')
tag = 'infoleak:automatic-detection="private-ssh-key"'
@ -127,7 +126,7 @@ class Keys(AbstractModule):
# find = True
if KeyEnum.SSH2_ENCRYPTED_PRIVATE_KEY.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has an ssh2 private key message')
print(f'{self.obj.get_global_id()} has an ssh2 private key message')
print('SSH2 private key message found')
tag = 'infoleak:automatic-detection="private-ssh-key"'
@ -135,7 +134,7 @@ class Keys(AbstractModule):
# find = True
if KeyEnum.OPENVPN_STATIC_KEY_V1.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has an openssh private key message')
print(f'{self.obj.get_global_id()} has an openssh private key message')
print('OpenVPN Static key message found')
tag = 'infoleak:automatic-detection="vpn-static-key"'
@ -143,21 +142,21 @@ class Keys(AbstractModule):
# find = True
if KeyEnum.DSA_PRIVATE_KEY.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has a dsa private key message')
print(f'{self.obj.get_global_id()} has a dsa private key message')
tag = 'infoleak:automatic-detection="dsa-private-key"'
self.add_message_to_queue(message=tag, queue='Tags')
# find = True
if KeyEnum.EC_PRIVATE_KEY.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has an ec private key message')
print(f'{self.obj.get_global_id()} has an ec private key message')
tag = 'infoleak:automatic-detection="ec-private-key"'
self.add_message_to_queue(message=tag, queue='Tags')
# find = True
if KeyEnum.PUBLIC_KEY.value in content:
self.redis_logger.warning(f'{self.obj.get_global_id()} has a public key message')
print(f'{self.obj.get_global_id()} has a public key message')
tag = 'infoleak:automatic-detection="public-key"'
self.add_message_to_queue(message=tag, queue='Tags')

10
bin/modules/LibInjection.py
View File

@ -34,10 +34,9 @@ class LibInjection(AbstractModule):
self.faup = Faup()
self.redis_logger.info(f"Module: {self.module_name} Launched")
self.logger.info(f"Module: {self.module_name} Launched")
def compute(self, message):
item = self.get_obj()
url = message
self.faup.decode(url)
@ -66,12 +65,9 @@ class LibInjection(AbstractModule):
# print(f'query is sqli : {result_query}')
if result_path['sqli'] is True or result_query['sqli'] is True:
item_id = item.get_id()
print(f"Detected (libinjection) SQL in URL: {item_id}")
print(unquote(url))
to_print = f'LibInjection;{item.get_source()};{item.get_date()};{item.get_basename()};Detected SQL in URL;{self.obj.get_global_id()}'
self.redis_logger.warning(to_print)
self.logger.info(f'Detected SQL in URL;{self.obj.get_global_id()}')
print(unquote(url))
# Add tag
tag = 'infoleak:automatic-detection="sql-injection"'

3
bin/modules/MISP_Thehive_Auto_Push.py
View File

@ -18,7 +18,6 @@ sys.path.append(os.environ['AIL_BIN'])
##################################
from modules.abstract_module import AbstractModule
from lib.exceptions import MISPConnectionError
from lib.objects.Items import Item
from lib import Tag
from exporter.MISPExporter import MISPExporterAutoDaily
from exporter.TheHiveExporter import TheHiveExporterAlertTag
@ -43,7 +42,7 @@ class MISP_Thehive_Auto_Push(AbstractModule):
if self.last_refresh < Tag.get_last_auto_push_refreshed() < 0:
self.tags = Tag.refresh_auto_push()
self.last_refresh = time.time()
self.redis_logger.info('Tags Auto Push refreshed')
self.logger.debug('Tags Auto Push refreshed')
tag = message
item = self.get_obj()

18
bin/modules/Mail.py
View File

@ -95,20 +95,20 @@ class Mail(AbstractModule):
# print()
except dns.resolver.NoNameservers:
self.redis_logger.debug('NoNameserver, No non-broken nameservers are available to answer the query.')
self.logger.debug('NoNameserver, No non-broken nameservers are available to answer the query.')
print('NoNameserver, No non-broken nameservers are available to answer the query.')
except dns.resolver.NoAnswer:
self.redis_logger.debug('NoAnswer, The response did not contain an answer to the question.')
self.logger.debug('NoAnswer, The response did not contain an answer to the question.')
print('NoAnswer, The response did not contain an answer to the question.')
except dns.name.EmptyLabel:
self.redis_logger.debug('SyntaxError: EmptyLabel')
self.logger.debug('SyntaxError: EmptyLabel')
print('SyntaxError: EmptyLabel')
except dns.resolver.NXDOMAIN:
# save_mxdomain_in_cache(mxdomain)
self.redis_logger.debug('The query name does not exist.')
self.logger.debug('The query name does not exist.')
print('The query name does not exist.')
except dns.name.LabelTooLong:
self.redis_logger.debug('The Label is too long')
self.logger.debug('The Label is too long')
print('The Label is too long')
except dns.exception.Timeout:
print('dns timeout')
@ -134,9 +134,7 @@ class Mail(AbstractModule):
# # TODO: sanitize mails
def compute(self, message):
score = message
item = self.get_obj()
item_date = item.get_date()
mails = self.regex_findall(self.email_regex, item.id, item.get_content())
mxdomains_email = {}
@ -171,15 +169,15 @@ class Mail(AbstractModule):
# for tld in mx_tlds:
# Statistics.add_module_tld_stats_by_date('mail', item_date, tld, mx_tlds[tld])
msg = f'Mails;{item.get_source()};{item_date};{item.get_basename()};Checked {num_valid_email} e-mail(s);{self.obj.get_global_id()}'
msg = f'Checked {num_valid_email} e-mail(s);{self.obj.get_global_id()}'
if num_valid_email > self.mail_threshold:
print(f'{item.id} Checked {num_valid_email} e-mail(s)')
self.redis_logger.warning(msg)
self.logger.info(msg)
# Tags
tag = 'infoleak:automatic-detection="mail"'
self.add_message_to_queue(message=tag, queue='Tags')
elif num_valid_email > 0:
self.redis_logger.info(msg)
self.logger.info(msg)
if __name__ == '__main__':

17
bin/modules/Onion.py
View File

@ -4,7 +4,7 @@
The Onion Module
============================
This module extract url from item and returning only ones which are tor
This module extract url from object and returning only ones which are tor
related (.onion). All These urls are send to the crawler discovery queue.
Requirements
@ -69,12 +69,11 @@ class Onion(AbstractModule):
onion_urls = []
domains = []
score = message
item = self.get_obj()
item_content = item.get_content()
obj = self.get_obj()
content = obj.get_content()
# max execution time on regex
res = self.regex_findall(self.onion_regex, item.get_id(), item_content)
res = self.regex_findall(self.onion_regex, obj.get_id(), content)
for x in res:
# String to tuple
x = x[2:-2].replace(" '", "").split("',")
@ -92,16 +91,14 @@ class Onion(AbstractModule):
if onion_urls:
if crawlers.is_crawler_activated():
for domain in domains:
task_uuid = crawlers.create_task(domain, parent=item.get_id(), priority=0,
task_uuid = crawlers.create_task(domain, parent=obj.get_id(), priority=0,
har=self.har, screenshot=self.screenshot)
if task_uuid:
print(f'{domain} added to crawler queue: {task_uuid}')
else:
to_print = f'Onion;{item.get_source()};{item.get_date()};{item.get_basename()};'
print(f'{to_print}Detected {len(domains)} .onion(s);{self.obj.get_global_id()}')
self.redis_logger.warning(f'{to_print}Detected {len(domains)} .onion(s);{self.obj.get_global_id()}')
print(f'Detected {len(domains)} .onion(s);{self.obj.get_global_id()}')
# TAG Item
# TAG Object
tag = 'infoleak:automatic-detection="onion"'
self.add_message_to_queue(message=tag, queue='Tags')

6
bin/modules/Phone.py
View File

@ -62,7 +62,7 @@ class Phone(AbstractModule):
tag = 'infoleak:automatic-detection="phone-number"'
self.add_message_to_queue(message=tag, queue='Tags')
self.redis_logger.warning(f'{self.obj.get_global_id()} contains {len(phone)} Phone numbers')
self.logger.info(f'{self.obj.get_global_id()} contains {len(phone)} Phone numbers')
# # List of the regex results in the Item, may be null
# results = self.REG_PHONE.findall(content)
@ -70,7 +70,7 @@ class Phone(AbstractModule):
# # If the list is greater than 4, we consider the Item may contain a list of phone numbers
# if len(results) > 4:
# self.logger.debug(results)
# self.redis_logger.warning(f'{item.get_id()} contains PID (phone numbers)')
# self.logger.info(f'{item.get_id()} contains PID (phone numbers)')
#
# msg = f'infoleak:automatic-detection="phone-number";{item.get_id()}'
# self.add_message_to_queue(msg, 'Tags')
@ -88,7 +88,7 @@ class Phone(AbstractModule):
# pass
# for country_code in stats:
# if stats[country_code] > 4:
# self.redis_logger.warning(f'{item.get_id()} contains Phone numbers with country code {country_code}')
# self.logger.info(f'{item.get_id()} contains Phone numbers with country code {country_code}')
if __name__ == '__main__':

17
bin/modules/SQLInjectionDetection.py
View File

@ -16,8 +16,7 @@ import sys
import re
import urllib.request
from datetime import datetime
from pyfaup.faup import Faup
# from pyfaup.faup import Faup
from urllib.parse import unquote
sys.path.append(os.environ['AIL_BIN'])
@ -25,8 +24,7 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages
##################################
from modules.abstract_module import AbstractModule
from lib.ConfigLoader import ConfigLoader
from lib.objects.Items import Item
# from lib.ConfigLoader import ConfigLoader
# from lib import Statistics
class SQLInjectionDetection(AbstractModule):
@ -39,22 +37,19 @@ class SQLInjectionDetection(AbstractModule):
def __init__(self):
super(SQLInjectionDetection, self).__init__()
self.faup = Faup()
# self.faup = Faup()
self.logger.info(f"Module: {self.module_name} Launched")
def compute(self, message):
url = message
item = self.get_obj()
if self.is_sql_injection(url):
self.faup.decode(url)
url_parsed = self.faup.get()
# self.faup.decode(url)
# url_parsed = self.faup.get()
print(f"Detected SQL in URL: {item.id}")
print(f"Detected SQL in URL: {self.obj.get_global_id()}")
print(urllib.request.unquote(url))
to_print = f'SQLInjection;{item.get_source()};{item.get_date()};{item.get_basename()};Detected SQL in URL;{self.obj.get_global_id()}'
self.redis_logger.warning(to_print)
# Tag
tag = f'infoleak:automatic-detection="sql-injection"'

4
bin/modules/SentimentAnalysis.py
View File

@ -167,11 +167,11 @@ class SentimentAnalysis(AbstractModule):
provider_timestamp = provider + '_' + str(timestamp)
self.db.incr('UniqID')
UniqID = self.db.get('UniqID')
self.redis_logger.debug(f'{provider_timestamp}->{UniqID}dropped{num_line_removed}lines')
self.logger.debug(f'{provider_timestamp}->{UniqID}dropped{num_line_removed}lines')
self.db.sadd(provider_timestamp, UniqID)
self.db.set(UniqID, avg_score)
else:
self.redis_logger.debug(f'Dropped:{p_MimeType}')
self.logger.debug(f'Dropped:{p_MimeType}')
def isJSON(self, content):

39
bin/modules/SubmitPaste.py
View File

@ -137,22 +137,21 @@ class SubmitPaste(AbstractModule):
"""
Create a paste for given file
"""
self.redis_logger.debug('manage')
if os.path.exists(file_full_path):
self.redis_logger.debug(f'file exists {file_full_path}')
self.logger.debug(f'file exists {file_full_path}')
file_size = os.stat(file_full_path).st_size
self.redis_logger.debug(f'file size {file_size}')
self.logger.debug(f'file size {file_size}')
# Verify file length
if file_size < SubmitPaste.FILE_MAX_SIZE:
# TODO sanitize filename
filename = file_full_path.split('/')[-1]
self.redis_logger.debug(f'sanitize filename {filename}')
self.redis_logger.debug('file size allowed')
self.logger.debug(f'sanitize filename {filename}')
self.logger.debug('file size allowed')
if not '.' in filename:
self.redis_logger.debug('no extension for filename')
self.logger.debug('no extension for filename')
try:
# Read file
with open(file_full_path,'r') as f:
@ -165,14 +164,14 @@ class SubmitPaste(AbstractModule):
else:
file_type = filename.rsplit('.', 1)[1]
file_type = file_type.lower()
self.redis_logger.debug(f'file ext {file_type}')
self.logger.debug(f'file ext {file_type}')
if file_type in SubmitPaste.ALLOWED_EXTENSIONS:
self.redis_logger.debug('Extension allowed')
self.logger.debug('Extension allowed')
# TODO enum of possible file extension ?
# TODO verify file hash with virus total ?
if not self._is_compressed_type(file_type):
self.redis_logger.debug('Plain text file')
self.logger.debug('Plain text file')
# plain txt file
with open(file_full_path,'r') as f:
content = f.read()
@ -197,7 +196,7 @@ class SubmitPaste(AbstractModule):
# except:
# self.abord_file_submission(uuid, "file decompression error")
# raise
# self.redis_logger.debug('unpacking {} file'.format(files.unpacker))
# self.logger.debug('unpacking {} file'.format(files.unpacker))
# if(not files.children):
# self.abord_file_submission(uuid, "Empty compressed file")
# raise
@ -209,11 +208,11 @@ class SubmitPaste(AbstractModule):
# self.create_paste(uuid, child.contents, ltags, ltagsgalaxies, uuid+'_'+ str(n) , source)
# n = n + 1
# else:
# self.redis_logger.error("Error in module %s: bad extention"%(self.module_name))
# self.logger.error("Error in module %s: bad extention"%(self.module_name))
# self.addError(uuid, 'Bad file extension: {}'.format(child.filename.decode()) )
# except FileNotFoundError:
# self.redis_logger.error("Error in module %s: file not found"%(self.module_name))
# self.logger.error("Error in module %s: file not found"%(self.module_name))
# self.addError(uuid, 'File not found: {}'.format(file_full_path), uuid )
else:
@ -248,7 +247,7 @@ class SubmitPaste(AbstractModule):
# delete uuid
self.r_serv_db.srem('submitted:uuid', uuid)
self.redis_logger.debug(f'{uuid} all file submitted')
self.logger.debug(f'{uuid} all file submitted')
print(f'{uuid} all file submitted')
def create_paste(self, uuid, paste_content, ltags, ltagsgalaxies, name, source=None):
@ -262,11 +261,11 @@ class SubmitPaste(AbstractModule):
full_path = os.path.join(ITEMS_FOLDER, save_path)
self.redis_logger.debug(f'file path of the paste {full_path}')
self.logger.debug(f'file path of the paste {full_path}')
if not os.path.isfile(full_path):
# file not exists in AIL paste directory
self.redis_logger.debug(f"new paste {paste_content}")
self.logger.debug(f"new paste {paste_content}")
gzip64encoded = self._compress_encode_content(paste_content, uuid)
@ -274,7 +273,7 @@ class SubmitPaste(AbstractModule):
# use relative path
rel_item_path = save_path.replace(self.PASTES_FOLDER, '', 1)
self.redis_logger.debug(f"relative path {rel_item_path}")
self.logger.debug(f"relative path {rel_item_path}")
item = Item(rel_item_path)
@ -295,12 +294,12 @@ class SubmitPaste(AbstractModule):
if self.r_serv_log_submit.get(f'{uuid}:nb_end') == self.r_serv_log_submit.get(f'{uuid}:nb_total'):
self.r_serv_log_submit.set(f'{uuid}:end', 1)
self.redis_logger.debug(f' {rel_item_path} send to Mixer')
self.logger.debug(f' {rel_item_path} send to Mixer')
print(f' {rel_item_path} send to Mixer')
self.r_serv_log_submit.sadd(f'{uuid}:paste_submit_link', rel_item_path)
curr_date = datetime.date.today()
self.redis_logger.debug("paste submitted")
self.logger.debug("paste submitted")
else:
self.addError(uuid, f'File: {save_path} already exist in submitted pastes')
@ -316,7 +315,7 @@ class SubmitPaste(AbstractModule):
return gzip64encoded
def addError(self, uuid, errorMessage):
self.redis_logger.debug(errorMessage)
self.logger.debug(errorMessage)
print(errorMessage)
error = self.r_serv_log_submit.get(f'{uuid}:error')
if error is not None:
@ -324,7 +323,7 @@ class SubmitPaste(AbstractModule):
self.r_serv_log_submit.incr(f'{uuid}:nb_end')
def abord_file_submission(self, uuid, errorMessage):
self.redis_logger.debug(f'abord {uuid}, {errorMessage}')
self.logger.debug(f'abord {uuid}, {errorMessage}')
self.addError(uuid, errorMessage)
self.r_serv_log_submit.set(f'{uuid}:end', 1)

2
bin/modules/Urls.py
View File

@ -85,8 +85,6 @@ class Urls(AbstractModule):
if len(l_urls) > 0:
to_print = f'Urls;{item.get_source()};{item.get_date()};{item.get_basename()};'
print(to_print)
# .debug ???
# self.redis_logger.info(f'{to_print}Detected {len(l_urls)} URL;{item.get_id()}')
if __name__ == '__main__':

10
bin/modules/abstract_module.py
View File

@ -18,7 +18,6 @@ sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from pubsublogger import publisher
from lib import ail_logger
from lib.ail_queues import AILQueue
from lib import regex_helper
@ -51,15 +50,6 @@ class AbstractModule(ABC):
self.obj = None
self.sha256_mess = None
# Init Redis Logger
self.redis_logger = publisher
# Port of the redis instance used by pubsublogger
self.redis_logger.port = 6380
# Channel name to publish logs
# # TODO: refactor logging
# If provided could be a namespaced channel like script:<ModuleName>
self.redis_logger.channel = 'Script'
# Cache key
self.r_cache_key = regex_helper.generate_redis_cache_key(self.module_name)
self.max_execution_time = 30

12
bin/trackers/Retro_Hunt.py
View File

@ -42,12 +42,11 @@ class Retro_Hunt_Module(AbstractModule):
self.obj = None
self.tags = []
self.redis_logger.info(f"Module: {self.module_name} Launched")
self.logger.info(f"Module: {self.module_name} Launched")
# # TODO: # start_time
# # end_time
def compute(self, task_uuid):
self.redis_logger.warning(f'{self.module_name}, starting Retro hunt task {task_uuid}')
print(f'starting Retro hunt task {task_uuid}')
self.progress = 0
# First launch
@ -58,7 +57,7 @@ class Retro_Hunt_Module(AbstractModule):
timeout = self.retro_hunt.get_timeout()
self.tags = self.retro_hunt.get_tags()
self.redis_logger.debug(f'{self.module_name}, Retro Hunt rule {task_uuid} timeout {timeout}')
self.logger.debug(f'{self.module_name}, Retro Hunt rule {task_uuid} timeout {timeout}')
# Filters
filters = self.retro_hunt.get_filters()
@ -111,7 +110,6 @@ class Retro_Hunt_Module(AbstractModule):
# Completed
self.retro_hunt.complete()
print(f'Retro Hunt {task_uuid} completed')
self.redis_logger.warning(f'{self.module_name}, Retro Hunt {task_uuid} completed')
def update_progress(self):
if self.nb_objs == 0:
@ -128,7 +126,6 @@ class Retro_Hunt_Module(AbstractModule):
# print(data)
task_uuid = data['namespace']
self.redis_logger.info(f'{self.module_name}, Retro hunt {task_uuid} match found: {obj_id}')
print(f'Retro hunt {task_uuid} match found: {self.obj.get_type()} {obj_id}')
self.retro_hunt.add(self.obj.get_type(), self.obj.get_subtype(r_str=True), obj_id)
@ -158,16 +155,15 @@ class Retro_Hunt_Module(AbstractModule):
task_uuid = Tracker.get_retro_hunt_task_to_start()
if task_uuid:
# Module processing with the message from the queue
self.redis_logger.debug(task_uuid)
self.logger.debug(task_uuid)
# try:
self.compute(task_uuid)
# except Exception as err:
# self.redis_logger.error(f'Error in module {self.module_name}: {err}')
# self.logger.error(f'Error in module {self.module_name}: {err}')
# # Remove uuid ref
# self.remove_submit_uuid(uuid)
else:
# Wait before next process
self.redis_logger.debug(f'{self.module_name}, waiting for new message, Idling {self.pending_seconds}s')
time.sleep(self.pending_seconds)

4
bin/trackers/Tracker_Regex.py
View File

@ -47,14 +47,13 @@ class Tracker_Regex(AbstractModule):
self.exporters = {'mail': MailExporterTracker(),
'webhook': WebHookExporterTracker()}
self.redis_logger.info(f"Module: {self.module_name} Launched")
self.logger.info(f"Module: {self.module_name} Launched")
def compute(self, message):
# refresh Tracked regex
if self.last_refresh < Tracker.get_tracker_last_updated_by_type('regex'):
self.tracked_regexs = Tracker.get_tracked_regexs()
self.last_refresh = time.time()
self.redis_logger.debug('Tracked regex refreshed')
print('Tracked regex refreshed')
obj = self.get_obj()
@ -117,7 +116,6 @@ class Tracker_Regex(AbstractModule):
continue
print(f'new tracked regex found: {tracker_name} in {self.obj.get_global_id()}')
self.redis_logger.warning(f'new tracked regex found: {tracker_name} in {self.obj.get_global_id()}')
tracker.add(obj.get_type(), obj.get_subtype(r_str=True), obj_id)

7
bin/trackers/Tracker_Term.py
View File

@ -62,20 +62,18 @@ class Tracker_Term(AbstractModule):
self.exporters = {'mail': MailExporterTracker(),
'webhook': WebHookExporterTracker()}
self.redis_logger.info(f"Module: {self.module_name} Launched")
self.logger.info(f"Module: {self.module_name} Launched")
def compute(self, message):
# refresh Tracked term
if self.last_refresh_word < Tracker.get_tracker_last_updated_by_type('word'):
self.tracked_words = Tracker.get_tracked_words()
self.last_refresh_word = time.time()
self.redis_logger.debug('Tracked word refreshed')
print('Tracked word refreshed')
if self.last_refresh_set < Tracker.get_tracker_last_updated_by_type('set'):
self.tracked_sets = Tracker.get_tracked_sets()
self.last_refresh_set = time.time()
self.redis_logger.debug('Tracked set refreshed')
print('Tracked set refreshed')
obj = self.get_obj()
@ -93,7 +91,7 @@ class Tracker_Term(AbstractModule):
try:
dict_words_freq = Tracker.get_text_word_frequency(content)
except TimeoutException:
self.redis_logger.warning(f"{self.obj.get_global_id()} processing timeout")
self.logger.warning(f"{self.obj.get_global_id()} processing timeout")
else:
signal.alarm(0)
@ -125,7 +123,6 @@ class Tracker_Term(AbstractModule):
continue
print(f'new tracked term {tracker_uuid} found: {tracker_name} in {self.obj.get_global_id()}')
self.redis_logger.warning(f'new tracked term found: {tracker_name} in {self.obj.get_global_id()}')
tracker.add(obj.get_type(), obj.get_subtype(), obj_id)