From ab24343b48b1b8f19e1a48fff61664aca0e0ab36 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 22 Feb 2023 11:08:29 +0100 Subject: [PATCH] fix: [modules] fix modules --- bin/LAUNCH.sh | 2 +- bin/exporter/MailExporter.py | 3 - bin/lib/Tracker.py | 112 ++++++++++++++++++++++++++- bin/lib/d4.py | 2 +- bin/lib/objects/Decodeds.py | 30 ++++--- bin/modules/ApiKey.py | 4 +- bin/modules/LibInjection.py | 3 - bin/modules/Phone.py | 3 - bin/modules/SQLInjectionDetection.py | 3 - bin/packages/modules.cfg | 2 +- 10 files changed, 131 insertions(+), 33 deletions(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index a16837ec..0977e5d4 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -217,7 +217,7 @@ function launching_scripts { sleep 0.1 screen -S "Script_AIL" -X screen -t "Cryptocurrency" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Cryptocurrencies.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Cve" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Cve.py; read x" + screen -S "Script_AIL" -X screen -t "CveModule" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./CveModule.py; read x" sleep 0.1 screen -S "Script_AIL" -X screen -t "Decoder" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Decoder.py; read x" sleep 0.1 diff --git a/bin/exporter/MailExporter.py b/bin/exporter/MailExporter.py index 97913645..c48e86cb 100755 --- a/bin/exporter/MailExporter.py +++ b/bin/exporter/MailExporter.py @@ -60,9 +60,6 @@ class MailExporter(AbstractExporter, ABC): self.port is None): raise Exception('SMTP configuration (host, port, sender) is missing or incomplete!') - def import(self): - pass - def get_smtp_client(self): # try: if self.pw is not None: diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index 49102249..45b7d059 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -26,7 +26,9 @@ from lib.Users import User config_loader = ConfigLoader.ConfigLoader() r_cache = config_loader.get_redis_conn("Redis_Cache") -r_serv_tracker = config_loader.get_db_conn("Kvrocks_Trackers") +r_tracker = config_loader.get_db_conn("Kvrocks_Trackers") + +r_serv_tracker = config_loader.get_db_conn("Kvrocks_Trackers") # TODO REMOVE ME items_dir = config_loader.get_config_str("Directories", "pastes") if items_dir[-1] == '/': @@ -68,12 +70,112 @@ def is_valid_mail(email): def verify_mail_list(mail_list): for mail in mail_list: if not is_valid_mail(mail): - return ({'status': 'error', 'reason': 'Invalid email', 'value': mail}, 400) + return {'status': 'error', 'reason': 'Invalid email', 'value': mail}, 400 return None ##-- UTILS --## ############### +################################################################################################ +################################################################################################ +################################################################################################ + +class Tracker: + def __init__(self, tracker_uuid): + self.uuid = tracker_uuid + + def get_uuid(self): + return self.uuid + + def exists(self): + return r_tracker.exists(f'tracker:{self.uuid}') + + def get_date(self): + return r_tracker.hget(f'tracker:{self.uuid}', 'date') + + def get_first_seen(self): + return r_tracker.hget(f'tracker:{self.uuid}', 'first_seen') + + def get_last_seen(self): + return r_tracker.hget(f'tracker:{self.uuid}', 'last_seen') + + def get_description(self): + return r_tracker.hget(f'tracker:{self.uuid}', 'description') + + def get_level(self): + level = r_tracker.hget(f'tracker:{self.uuid}', 'level') + if not level: + level = 0 + return int(level) + + def get_sources(self): + return r_tracker.smembers(f'tracker:sources:{self.uuid}') + + def get_tracker(self): + return r_serv_tracker.hget(f'tracker:{self.uuid}', 'tracked') + + def get_type(self): + return r_tracker.hget(f'tracker:{self.uuid}', 'type') + + def get_tags(self): + return r_tracker.smembers(f'tracker:tags:{self.uuid}') + + def mail_export(self): + return r_tracker.exists(f'tracker:mail:{self.uuid}') + + def get_mails(self): + return r_tracker.smembers(f'tracker:mail:{self.uuid}') + + def get_user(self): + return r_tracker.hget(f'tracker:{self.uuid}', 'user_id') + + def webhook_export(self): + return r_tracker.hexists(f'tracker:mail:{self.uuid}', 'webhook') + + def get_webhook(self): + return r_tracker.hget(f'tracker:{self.uuid}', 'webhook') + + # TODO get objects/ tracked items + + + # TODO sparkline + def get_meta(self, options): + if not options: + options = set() + meta = {'uuid': self.uuid, + 'tracker': self.get_tracker(), + 'type': self.get_type(), + 'date': self.get_date(), + 'first_seen': self.get_first_seen(), + 'last_seen': self.get_last_seen()} + if 'user' in options: + meta['user'] = self.get_user() + if 'level' in options: + meta['level'] = self.get_level() + if 'description' in options: + meta['description'] = self.get_description() + if 'tags' in options: + meta['tags'] = self.get_tags() + if 'sources' in options: + meta['sources'] = self.get_sources() + if 'mails' in options: + meta['mails'] = self.get_mails() + if 'webhooks' in options: + meta['webhook'] = self.get_webhook() + # if 'sparkline' in options: + # meta['sparkline'] = get_tracker_sparkline(tracker_uuid) + + + + + # TODO + def add(self, obj_id): + pass + +################################################################################################ +################################################################################################ +################################################################################################ + def get_all_tracker_type(): return ['word', 'set', 'regex', 'yara'] @@ -345,7 +447,7 @@ def is_tracker_in_user_level(tracker, tracker_type, user_id): ## API ## def api_check_tracker_uuid(tracker_uuid): - if not is_valid_uuid_v4(task_uuid): + if not is_valid_uuid_v4(tracker_uuid): return {"status": "error", "reason": "Invalid uuid"}, 400 if not r_serv_tracker.exists(f'tracker:{tracker_uuid}'): return {"status": "error", "reason": "Unknown uuid"}, 404 @@ -678,6 +780,10 @@ def reload_yara_rules(): l_tracker_uuid = get_tracker_uuid_list(yar_path, 'yara') for tracker_uuid in l_tracker_uuid: rule_dict[tracker_uuid] = os.path.join(get_yara_rules_dir(), yar_path) + for tracker_uuid in rule_dict: + if not os.path.isfile(rule_dict[tracker_uuid]): + # TODO IGNORE + LOGS + raise Exception(f"Error: {rule_dict[tracker_uuid]} doesn't exists") rules = yara.compile(filepaths=rule_dict) return rules diff --git a/bin/lib/d4.py b/bin/lib/d4.py index f8e57d2f..561e635c 100755 --- a/bin/lib/d4.py +++ b/bin/lib/d4.py @@ -40,7 +40,7 @@ def is_passive_dns_enabled(cache=True): res = r_cache.get('d4:passivedns:enabled') if res is None: res = r_serv_db.hget('d4:passivedns', 'enabled') == 'True' - r_cache.set('d4:passivedns:enabled', res) + r_cache.set('d4:passivedns:enabled', str(res)) return res else: return res == 'True' diff --git a/bin/lib/objects/Decodeds.py b/bin/lib/objects/Decodeds.py index e826b48c..0f5a5a31 100755 --- a/bin/lib/objects/Decodeds.py +++ b/bin/lib/objects/Decodeds.py @@ -3,6 +3,7 @@ import os import sys +import magic import requests import zipfile @@ -135,9 +136,9 @@ class Decoded(AbstractDaterangeObject): obj.first_seen = self.get_first_seen() obj.last_seen = self.get_last_seen() - obj_attrs.append( obj.add_attribute('sha1', value=self.id)) - obj_attrs.append( obj.add_attribute('mimetype', value=self.get_mimetype())) - obj_attrs.append( obj.add_attribute('malware-sample', value=self.id, data=self.get_content())) + obj_attrs.append(obj.add_attribute('sha1', value=self.id)) + obj_attrs.append(obj.add_attribute('mimetype', value=self.get_mimetype())) + obj_attrs.append(obj.add_attribute('malware-sample', value=self.id, data=self.get_content())) for obj_attr in obj_attrs: for tag in self.get_tags(): obj_attr.add_tag(tag) @@ -177,7 +178,10 @@ class Decoded(AbstractDaterangeObject): else: return {} + # TODO def guess_mimetype(self, bytes_content): + # if not bytes_content: + # bytes_content = self.get_content() return magic.from_buffer(bytes_content, mime=True) # avoid counting the same hash multiple time on the same item @@ -226,15 +230,15 @@ class Decoded(AbstractDaterangeObject): # -> sinter with r_objects.sunion(f'decoded:algo:{algo_name}:{date}') # # TODO: ADD items - def create(self, content, date, mimetype=None): - if not mimetype: - mimetype = self.guess_mimetype(content) - self.save_file(content, mimetype) - - - update_decoded_daterange(sha1_string, date_from) - if date_from != date_to and date_to: - update_decoded_daterange(sha1_string, date_to) + # def create(self, content, date, mimetype=None): + # if not mimetype: + # mimetype = self.guess_mimetype(content) + # self.save_file(content, mimetype) + # + # + # update_decoded_daterange(sha1_string, date_from) + # if date_from != date_to and date_to: + # update_decoded_daterange(sha1_string, date_to) ####################################################################################### ####################################################################################### @@ -266,6 +270,8 @@ class Decoded(AbstractDaterangeObject): # file in queue elif response_code == -2: report = 'In Queue - Refresh' + else: + report = 'Error - Unknown VT response' self.set_vt_report(report) print(json_response) print(response_code) diff --git a/bin/modules/ApiKey.py b/bin/modules/ApiKey.py index 400e09c4..6b2cf9b3 100755 --- a/bin/modules/ApiKey.py +++ b/bin/modules/ApiKey.py @@ -39,6 +39,7 @@ class ApiKey(AbstractModule): re.compile(self.re_aws_secret_key) # r'=AIza[0-9a-zA-Z-_]{35}' keep equal ???? + # AIza[0-9A-Za-z\\-_]{35} self.re_google_api_key = r'AIza[0-9a-zA-Z-_]{35}' re.compile(self.re_google_api_key) @@ -80,9 +81,6 @@ class ApiKey(AbstractModule): msg = f'infoleak:automatic-detection="api-key";{item.get_id()}' self.send_message_to_queue(msg, 'Tags') - # Send to duplicate - self.send_message_to_queue(item.get_id(), 'Duplicate') - if r_result: return google_api_key, aws_access_key, aws_secret_key diff --git a/bin/modules/LibInjection.py b/bin/modules/LibInjection.py index 24c7ca23..7893da0f 100755 --- a/bin/modules/LibInjection.py +++ b/bin/modules/LibInjection.py @@ -76,9 +76,6 @@ class LibInjection(AbstractModule): to_print = f'LibInjection;{item.get_source()};{item.get_date()};{item.get_basename()};Detected SQL in URL;{item_id}' self.redis_logger.warning(to_print) - # Send to duplicate - self.send_message_to_queue(item_id, 'Duplicate') - # Add tag msg = f'infoleak:automatic-detection="sql-injection";{item_id}' self.send_message_to_queue(msg, 'Tags') diff --git a/bin/modules/Phone.py b/bin/modules/Phone.py index a7d2a87d..3a936105 100755 --- a/bin/modules/Phone.py +++ b/bin/modules/Phone.py @@ -56,9 +56,6 @@ class Phone(AbstractModule): msg = f'infoleak:automatic-detection="phone-number";{item.get_id()}' self.send_message_to_queue(msg, 'Tags') - # Send to duplicate - self.send_message_to_queue(item.get_id(), 'Duplicate') - stats = {} for phone_number in results: try: diff --git a/bin/modules/SQLInjectionDetection.py b/bin/modules/SQLInjectionDetection.py index 2cc11066..77f78cff 100755 --- a/bin/modules/SQLInjectionDetection.py +++ b/bin/modules/SQLInjectionDetection.py @@ -57,9 +57,6 @@ class SQLInjectionDetection(AbstractModule): to_print = f'SQLInjection;{item.get_source()};{item.get_date()};{item.get_basename()};Detected SQL in URL;{item_id}' self.redis_logger.warning(to_print) - # Send to duplicate - self.send_message_to_queue(item_id, 'Duplicate') - # Tag msg = f'infoleak:automatic-detection="sql-injection";{item_id}' self.send_message_to_queue(msg, 'Tags') diff --git a/bin/packages/modules.cfg b/bin/packages/modules.cfg index 1c718dab..5c65b605 100644 --- a/bin/packages/modules.cfg +++ b/bin/packages/modules.cfg @@ -36,7 +36,7 @@ publish = Redis_Host subscribe = Redis_Host publish = Redis_D4_client -[D4_client] +[D4Client] subscribe = Redis_D4_client [Retro_Hunt]