CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

fix: [modules] fix modules

pull/594/head
Terrtia 2023-02-22 11:08:29 +01:00
parent 6842efc15d
commit ab24343b48
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
10 changed files with 131 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bin/LAUNCH.sh
View File

@ -217,7 +217,7 @@ function launching_scripts {
sleep 0.1
screen -S "Script_AIL" -X screen -t "Cryptocurrency" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Cryptocurrencies.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Cve" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Cve.py; read x"
screen -S "Script_AIL" -X screen -t "CveModule" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./CveModule.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Decoder" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Decoder.py; read x"
sleep 0.1

3
bin/exporter/MailExporter.py
View File

@ -60,9 +60,6 @@ class MailExporter(AbstractExporter, ABC):
self.port is None):
raise Exception('SMTP configuration (host, port, sender) is missing or incomplete!')
def import(self):
pass
def get_smtp_client(self):
# try:
if self.pw is not None:

112
bin/lib/Tracker.py
View File

@ -26,7 +26,9 @@ from lib.Users import User
config_loader = ConfigLoader.ConfigLoader()
r_cache = config_loader.get_redis_conn("Redis_Cache")
r_serv_tracker = config_loader.get_db_conn("Kvrocks_Trackers")
r_tracker = config_loader.get_db_conn("Kvrocks_Trackers")
r_serv_tracker = config_loader.get_db_conn("Kvrocks_Trackers") # TODO REMOVE ME
items_dir = config_loader.get_config_str("Directories", "pastes")
if items_dir[-1] == '/':
@ -68,12 +70,112 @@ def is_valid_mail(email):
def verify_mail_list(mail_list):
for mail in mail_list:
if not is_valid_mail(mail):
return ({'status': 'error', 'reason': 'Invalid email', 'value': mail}, 400)
return {'status': 'error', 'reason': 'Invalid email', 'value': mail}, 400
return None
##-- UTILS --##
###############
################################################################################################
################################################################################################
################################################################################################
class Tracker:
def __init__(self, tracker_uuid):
self.uuid = tracker_uuid
def get_uuid(self):
return self.uuid
def exists(self):
return r_tracker.exists(f'tracker:{self.uuid}')
def get_date(self):
return r_tracker.hget(f'tracker:{self.uuid}', 'date')
def get_first_seen(self):
return r_tracker.hget(f'tracker:{self.uuid}', 'first_seen')
def get_last_seen(self):
return r_tracker.hget(f'tracker:{self.uuid}', 'last_seen')
def get_description(self):
return r_tracker.hget(f'tracker:{self.uuid}', 'description')
def get_level(self):
level = r_tracker.hget(f'tracker:{self.uuid}', 'level')
if not level:
level = 0
return int(level)
def get_sources(self):
return r_tracker.smembers(f'tracker:sources:{self.uuid}')
def get_tracker(self):
return r_serv_tracker.hget(f'tracker:{self.uuid}', 'tracked')
def get_type(self):
return r_tracker.hget(f'tracker:{self.uuid}', 'type')
def get_tags(self):
return r_tracker.smembers(f'tracker:tags:{self.uuid}')
def mail_export(self):
return r_tracker.exists(f'tracker:mail:{self.uuid}')
def get_mails(self):
return r_tracker.smembers(f'tracker:mail:{self.uuid}')
def get_user(self):
return r_tracker.hget(f'tracker:{self.uuid}', 'user_id')
def webhook_export(self):
return r_tracker.hexists(f'tracker:mail:{self.uuid}', 'webhook')
def get_webhook(self):
return r_tracker.hget(f'tracker:{self.uuid}', 'webhook')
# TODO get objects/ tracked items
# TODO sparkline
def get_meta(self, options):
if not options:
options = set()
meta = {'uuid': self.uuid,
'tracker': self.get_tracker(),
'type': self.get_type(),
'date': self.get_date(),
'first_seen': self.get_first_seen(),
'last_seen': self.get_last_seen()}
if 'user' in options:
meta['user'] = self.get_user()
if 'level' in options:
meta['level'] = self.get_level()
if 'description' in options:
meta['description'] = self.get_description()
if 'tags' in options:
meta['tags'] = self.get_tags()
if 'sources' in options:
meta['sources'] = self.get_sources()
if 'mails' in options:
meta['mails'] = self.get_mails()
if 'webhooks' in options:
meta['webhook'] = self.get_webhook()
# if 'sparkline' in options:
# meta['sparkline'] = get_tracker_sparkline(tracker_uuid)
# TODO
def add(self, obj_id):
pass
################################################################################################
################################################################################################
################################################################################################
def get_all_tracker_type():
return ['word', 'set', 'regex', 'yara']
@ -345,7 +447,7 @@ def is_tracker_in_user_level(tracker, tracker_type, user_id):
## API ##
def api_check_tracker_uuid(tracker_uuid):
if not is_valid_uuid_v4(task_uuid):
if not is_valid_uuid_v4(tracker_uuid):
return {"status": "error", "reason": "Invalid uuid"}, 400
if not r_serv_tracker.exists(f'tracker:{tracker_uuid}'):
return {"status": "error", "reason": "Unknown uuid"}, 404
@ -678,6 +780,10 @@ def reload_yara_rules():
l_tracker_uuid = get_tracker_uuid_list(yar_path, 'yara')
for tracker_uuid in l_tracker_uuid:
rule_dict[tracker_uuid] = os.path.join(get_yara_rules_dir(), yar_path)
for tracker_uuid in rule_dict:
if not os.path.isfile(rule_dict[tracker_uuid]):
# TODO IGNORE + LOGS
raise Exception(f"Error: {rule_dict[tracker_uuid]} doesn't exists")
rules = yara.compile(filepaths=rule_dict)
return rules

2
bin/lib/d4.py
View File

@ -40,7 +40,7 @@ def is_passive_dns_enabled(cache=True):
res = r_cache.get('d4:passivedns:enabled')
if res is None:
res = r_serv_db.hget('d4:passivedns', 'enabled') == 'True'
r_cache.set('d4:passivedns:enabled', res)
r_cache.set('d4:passivedns:enabled', str(res))
return res
else:
return res == 'True'

30
bin/lib/objects/Decodeds.py
View File

@ -3,6 +3,7 @@
import os
import sys
import magic
import requests
import zipfile
@ -135,9 +136,9 @@ class Decoded(AbstractDaterangeObject):
obj.first_seen = self.get_first_seen()
obj.last_seen = self.get_last_seen()
obj_attrs.append( obj.add_attribute('sha1', value=self.id))
obj_attrs.append( obj.add_attribute('mimetype', value=self.get_mimetype()))
obj_attrs.append( obj.add_attribute('malware-sample', value=self.id, data=self.get_content()))
obj_attrs.append(obj.add_attribute('sha1', value=self.id))
obj_attrs.append(obj.add_attribute('mimetype', value=self.get_mimetype()))
obj_attrs.append(obj.add_attribute('malware-sample', value=self.id, data=self.get_content()))
for obj_attr in obj_attrs:
for tag in self.get_tags():
obj_attr.add_tag(tag)
@ -177,7 +178,10 @@ class Decoded(AbstractDaterangeObject):
else:
return {}
# TODO
def guess_mimetype(self, bytes_content):
# if not bytes_content:
# bytes_content = self.get_content()
return magic.from_buffer(bytes_content, mime=True)
# avoid counting the same hash multiple time on the same item
@ -226,15 +230,15 @@ class Decoded(AbstractDaterangeObject):
# -> sinter with r_objects.sunion(f'decoded:algo:{algo_name}:{date}')
# # TODO: ADD items
def create(self, content, date, mimetype=None):
if not mimetype:
mimetype = self.guess_mimetype(content)
self.save_file(content, mimetype)
update_decoded_daterange(sha1_string, date_from)
if date_from != date_to and date_to:
update_decoded_daterange(sha1_string, date_to)
# def create(self, content, date, mimetype=None):
# if not mimetype:
# mimetype = self.guess_mimetype(content)
# self.save_file(content, mimetype)
#
#
# update_decoded_daterange(sha1_string, date_from)
# if date_from != date_to and date_to:
# update_decoded_daterange(sha1_string, date_to)
#######################################################################################
#######################################################################################
@ -266,6 +270,8 @@ class Decoded(AbstractDaterangeObject):
# file in queue
elif response_code == -2:
report = 'In Queue - Refresh'
else:
report = 'Error - Unknown VT response'
self.set_vt_report(report)
print(json_response)
print(response_code)

4
bin/modules/ApiKey.py
View File

@ -39,6 +39,7 @@ class ApiKey(AbstractModule):
re.compile(self.re_aws_secret_key)
# r'=AIza[0-9a-zA-Z-_]{35}' keep equal ????
# AIza[0-9A-Za-z\\-_]{35}
self.re_google_api_key = r'AIza[0-9a-zA-Z-_]{35}'
re.compile(self.re_google_api_key)
@ -80,9 +81,6 @@ class ApiKey(AbstractModule):
msg = f'infoleak:automatic-detection="api-key";{item.get_id()}'
self.send_message_to_queue(msg, 'Tags')
# Send to duplicate
self.send_message_to_queue(item.get_id(), 'Duplicate')
if r_result:
return google_api_key, aws_access_key, aws_secret_key

3
bin/modules/LibInjection.py
View File

@ -76,9 +76,6 @@ class LibInjection(AbstractModule):
to_print = f'LibInjection;{item.get_source()};{item.get_date()};{item.get_basename()};Detected SQL in URL;{item_id}'
self.redis_logger.warning(to_print)
# Send to duplicate
self.send_message_to_queue(item_id, 'Duplicate')
# Add tag
msg = f'infoleak:automatic-detection="sql-injection";{item_id}'
self.send_message_to_queue(msg, 'Tags')

3
bin/modules/Phone.py
View File

@ -56,9 +56,6 @@ class Phone(AbstractModule):
msg = f'infoleak:automatic-detection="phone-number";{item.get_id()}'
self.send_message_to_queue(msg, 'Tags')
# Send to duplicate
self.send_message_to_queue(item.get_id(), 'Duplicate')
stats = {}
for phone_number in results:
try:

3
bin/modules/SQLInjectionDetection.py
View File

@ -57,9 +57,6 @@ class SQLInjectionDetection(AbstractModule):
to_print = f'SQLInjection;{item.get_source()};{item.get_date()};{item.get_basename()};Detected SQL in URL;{item_id}'
self.redis_logger.warning(to_print)
# Send to duplicate
self.send_message_to_queue(item_id, 'Duplicate')
# Tag
msg = f'infoleak:automatic-detection="sql-injection";{item_id}'
self.send_message_to_queue(msg, 'Tags')

2
bin/packages/modules.cfg
View File

@ -36,7 +36,7 @@ publish = Redis_Host
subscribe = Redis_Host
publish = Redis_D4_client
[D4_client]
[D4Client]
subscribe = Redis_D4_client
[Retro_Hunt]