#!/usr/bin/env python3 # -*-coding:UTF-8 -* import json import os import logging import logging.config import re import sys import time import uuid import yara import datetime import base64 from ail_typo_squatting import runAll import math from collections import defaultdict from flask import escape from textblob import TextBlob from nltk.tokenize import RegexpTokenizer sys.path.append(os.environ['AIL_BIN']) ################################## # Import Project packages ################################## from packages import Date from lib.ail_core import get_objects_tracked, get_object_all_subtypes, get_objects_retro_hunted from lib import ail_logger from lib import ConfigLoader from lib import item_basic from lib import Tag from lib.Users import User # LOGS logging.config.dictConfig(ail_logger.get_config(name='modules')) logger = logging.getLogger() config_loader = ConfigLoader.ConfigLoader() r_cache = config_loader.get_redis_conn("Redis_Cache") r_tracker = config_loader.get_db_conn("Kvrocks_Trackers") items_dir = config_loader.get_config_str("Directories", "pastes") if items_dir[-1] == '/': items_dir = items_dir[:-1] config_loader = None email_regex = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}' email_regex = re.compile(email_regex) special_characters = set('[<>~!?@#$%^&*|()_-+={}":;,.\'\n\r\t]/\\') special_characters.add('\\s') # NLTK tokenizer tokenizer = RegexpTokenizer('[\&\~\:\;\,\.\(\)\{\}\|\[\]\\\\/\-/\=\'\"\%\$\?\@\+\#\_\^\<\>\!\*\n\r\t\s]+', gaps=True, discard_empty=True) ############### #### UTILS #### def is_valid_uuid_v4(curr_uuid): if not curr_uuid: return False curr_uuid = curr_uuid.replace('-', '') try: uuid_test = uuid.UUID(hex=curr_uuid, version=4) return uuid_test.hex == curr_uuid except: return False def is_valid_regex(tracker_regex): try: re.compile(tracker_regex) return True except: return False def is_valid_mail(email): result = email_regex.match(email) if result: return True else: return False def verify_mail_list(mail_list): for mail in mail_list: if not is_valid_mail(mail): return {'status': 'error', 'reason': 'Invalid email', 'value': mail}, 400 return None ## -- UTILS -- ## ################# ################### #### TRACKER #### class Tracker: def __init__(self, tracker_uuid): self.uuid = tracker_uuid def get_uuid(self): return self.uuid def exists(self): return r_tracker.exists(f'tracker:{self.uuid}') def _get_field(self, field): return r_tracker.hget(f'tracker:{self.uuid}', field) def _set_field(self, field, value): r_tracker.hset(f'tracker:{self.uuid}', field, value) def get_date(self): return self._get_field('date') def get_last_change(self, r_str=False): last_change = self._get_field('last_change') if r_str and last_change: last_change = datetime.datetime.fromtimestamp(float(last_change)).strftime('%Y-%m-%d %H:%M:%S') return last_change def get_first_seen(self): return self._get_field('first_seen') def get_last_seen(self): return self._get_field('last_seen') def _set_first_seen(self, date): self._set_field('first_seen', date) def _set_last_seen(self, date): self._set_field('last_seen', date) def _exist_date(self, date): return r_tracker.exists(f'tracker:objs:{self.uuid}:{date}') # TODO: ADD CACHE ??? def update_daterange(self, date=None): first_seen = self.get_first_seen() # Added Object if date: date = int(date) first_seen = self.get_first_seen() # if op == 'add': if not first_seen: self._set_first_seen(date) self._set_last_seen(date) else: first_seen = int(first_seen) last_seen = int(self.get_last_seen()) if date < first_seen: self._set_first_seen(date) if date > last_seen: self._set_last_seen(date) else: last_seen = self.get_last_seen() if first_seen and last_seen: valid_first_seen = self._exist_date(first_seen) valid_last_seen = self._exist_date(last_seen) # update first seen if not valid_first_seen: for date in Date.get_daterange(first_seen, last_seen): if self._exist_date(date): self._set_first_seen(date) valid_first_seen = True break # update last seen if not valid_last_seen: for date in reversed(Date.get_daterange(first_seen, last_seen)): if self._exist_date(date): self._set_first_seen(date) valid_last_seen = True break if not valid_first_seen or not valid_last_seen: r_tracker.hdel(f'tracker:{self.uuid}', 'first_seen') r_tracker.hdel(f'tracker:{self.uuid}', 'last_seen') def get_description(self): return self._get_field('description') def get_level(self): level = self._get_field('level') if not level: level = 0 return int(level) def is_level_user(self): return self.get_level() == 0 def is_level_global(self): return self.get_level() == 1 def _set_level(self, level, tracker_type=None, user=None): if not tracker_type: tracker_type = self.get_type() if level == 0: # user only if not user: user = self.get_user() r_tracker.sadd(f'user:tracker:{user}', self.uuid) r_tracker.sadd(f'user:tracker:{user}:{tracker_type}', self.uuid) elif level == 1: # global r_tracker.sadd('global:tracker', self.uuid) r_tracker.sadd(f'global:tracker:{tracker_type}', self.uuid) self._set_field('level', level) def get_filters(self): filters = self._get_field('filters') if not filters: return {} else: return json.loads(filters) def set_filters(self, filters): if filters: self._set_field('filters', json.dumps(filters)) def del_filters(self, tracker_type, to_track): filters = self.get_filters() for obj_type in filters: r_tracker.srem(f'trackers:objs:{tracker_type}:{obj_type}', to_track) r_tracker.srem(f'trackers:uuid:{tracker_type}:{to_track}', f'{self.uuid}:{obj_type}') r_tracker.hdel(f'tracker:{self.uuid}', 'filters') def get_tracked(self): return self._get_field('tracked') def get_type(self): return self._get_field('type') def get_tags(self): return r_tracker.smembers(f'tracker:tags:{self.uuid}') def _set_tags(self, tags): for tag in tags: r_tracker.sadd(f'tracker:tags:{self.uuid}', tag) Tag.create_custom_tag(tag) # TODO CUSTOM TAGS def _del_tags(self): return r_tracker.delete(f'tracker:tags:{self.uuid}') def mail_export(self): return r_tracker.exists(f'tracker:mail:{self.uuid}') def get_mails(self): return r_tracker.smembers(f'tracker:mail:{self.uuid}') def _set_mails(self, mails): for mail in mails: r_tracker.sadd(f'tracker:mail:{self.uuid}', escape(mail)) def _del_mails(self): r_tracker.delete(f'tracker:mail:{self.uuid}') def get_user(self): return self._get_field('user_id') def webhook_export(self): webhook = self.get_webhook() return webhook is not None and webhook def get_webhook(self): return r_tracker.hget(f'tracker:{self.uuid}', 'webhook') def get_sparkline(self, nb_day=6): date_range_sparkline = Date.get_date_range(nb_day) sparkline = [] for date in date_range_sparkline: nb_seen_this_day = self.get_nb_objs_by_date(date) if nb_seen_this_day is None: nb_seen_this_day = 0 sparkline.append(int(nb_seen_this_day)) return sparkline def get_rule(self): yar_path = self.get_tracked() return yara.compile(filepath=os.path.join(get_yara_rules_dir(), yar_path)) def get_meta(self, options): if not options: options = set() meta = {'uuid': self.uuid, 'tracked': self.get_tracked(), # TODO TO CHECK 'type': self.get_type(), 'date': self.get_date(), 'first_seen': self.get_first_seen(), 'last_seen': self.get_last_seen()} if 'user' in options: meta['user'] = self.get_user() if 'level' in options: meta['level'] = self.get_level() if 'description' in options: meta['description'] = self.get_description() if 'nb_objs' in options: meta['nb_objs'] = self.get_nb_objs() if 'tags' in options: meta['tags'] = self.get_tags() if 'filters' in options: meta['filters'] = self.get_filters() if 'mails' in options: meta['mails'] = self.get_mails() if 'webhooks' in options: meta['webhook'] = self.get_webhook() if 'sparkline' in options: meta['sparkline'] = self.get_sparkline(6) return meta def _add_to_dashboard(self, obj_type, subtype, obj_id): mess = f'{self.uuid}:{int(time.time())}:{obj_type}:{subtype}:{obj_id}' if self.is_level_user(): user = self.get_user() r_tracker.lpush(f'trackers:user:{user}', mess) r_tracker.ltrim(f'trackers:user:{user}', 0, 9) else: r_tracker.lpush('trackers:dashboard', mess) r_tracker.ltrim(f'trackers:dashboard', 0, 9) def get_nb_objs_by_type(self, obj_type): return r_tracker.scard(f'tracker:objs:{self.uuid}:{obj_type}') def get_objs_by_type(self, obj_type): return r_tracker.smembers(f'tracker:objs:{self.uuid}:{obj_type}') def get_nb_objs(self): objs = {} for obj_type in get_objects_tracked(): nb = self.get_nb_objs_by_type(obj_type) if nb: objs[obj_type] = nb return objs def get_objs(self): objs = [] for obj_type in get_objects_tracked(): for obj in self.get_objs_by_type(obj_type): subtype, obj_id = obj.split(':', 1) objs.append((obj_type, subtype, obj_id)) return objs def get_nb_objs_by_date(self, date): return r_tracker.scard(f'tracker:objs:{self.uuid}:{date}') def get_objs_by_date(self, date): return r_tracker.smembers(f'tracker:objs:{self.uuid}:{date}') def get_objs_by_daterange(self, date_from, date_to): objs = set() for date in Date.get_daterange(date_from, date_to): objs |= self.get_objs_by_date(date) return objs def get_obj_dates(self, obj_type, subtype, obj_id): return r_tracker.smembers(f'obj:tracker:{obj_type}:{subtype}:{obj_id}:{self.uuid}') # - TODO Data Retention TO Implement - # # Or Daily/Monthly Global DB Cleanup: # Iterate on each tracker: # Iterate on each Obj: # Iterate on each date: # Delete from tracker range if date limit exceeded # - TODO def add(self, obj_type, subtype, obj_id, date=None): if not subtype: subtype = '' if not date: date = Date.get_today_date_str() new_obj_date = r_tracker.sadd(f'tracker:objs:{self.uuid}:{date}', f'{obj_type}:{subtype}:{obj_id}') r_tracker.sadd(f'obj:trackers:{obj_type}:{subtype}:{obj_id}', self.uuid) # Only save object match date - Needed for the DB Cleaner r_tracker.sadd(f'obj:tracker:{obj_type}:{subtype}:{obj_id}:{self.uuid}', date) r_tracker.sadd(f'tracker:objs:{self.uuid}:{obj_type}', f'{subtype}:{obj_id}') if new_obj_date: self.update_daterange(date) self._add_to_dashboard(obj_type, subtype, obj_id) def remove(self, obj_type, subtype, obj_id): if not subtype: subtype = '' for date in self.get_obj_dates(obj_type, subtype, obj_id): r_tracker.srem(f'tracker:objs:{self.uuid}:{date}', f'{obj_type}:{subtype}:{obj_id}') r_tracker.srem(f'obj:tracker:{obj_type}:{subtype}:{obj_id}:{self.uuid}', date) r_tracker.srem(f'obj:trackers:{obj_type}:{subtype}:{obj_id}', self.uuid) r_tracker.srem(f'tracker:objs:{self.uuid}', f'{obj_type}:{subtype}:{obj_id}') self.update_daterange() # TODO escape custom tags # TODO escape mails ???? def create(self, tracker_type, to_track, user_id, level, description=None, filters={}, tags=[], mails=[], webhook=None): if self.exists(): raise Exception('Error: Tracker already exists') # YARA if tracker_type == 'yara_custom' or tracker_type == 'yara_default': to_track = save_yara_rule(tracker_type, to_track, tracker_uuid=self.uuid) tracker_type = 'yara' elif tracker_type == 'typosquatting': domain = to_track.split(" ")[0] typo_generation = runAll(domain=domain, limit=math.inf, formatoutput="text", pathOutput="-", verbose=False) # TODO REPLACE LIMIT BY -1 for typo in typo_generation: r_tracker.sadd(f'tracker:typosquatting:{to_track}', typo) # create metadata self._set_field('tracked', to_track) self._set_field('type', tracker_type) self._set_field('date', datetime.date.today().strftime("%Y%m%d")) self._set_field('user_id', user_id) if description: self._set_field('description', escape(description)) if webhook: self._set_field('webhook', webhook) # create all tracker set r_tracker.sadd(f'all:tracker:{tracker_type}', to_track) # create tracker - uuid map r_tracker.sadd(f'all:tracker_uuid:{tracker_type}:{to_track}', self.uuid) r_tracker.sadd('trackers:all', self.uuid) r_tracker.sadd(f'trackers:all:{tracker_type}', self.uuid) # TRACKER LEVEL self._set_level(level, tracker_type=tracker_type, user=user_id) # create tracker tags list if tags: self._set_tags(tags) # create tracker mail notification list if mails: self._set_mails(mails) # Filters if not filters: filters = {} for obj_type in get_objects_tracked(): filters[obj_type] = {} else: self.set_filters(filters) for obj_type in filters: r_tracker.sadd(f'trackers:objs:{tracker_type}:{obj_type}', to_track) r_tracker.sadd(f'trackers:uuid:{tracker_type}:{to_track}', f'{self.uuid}:{obj_type}') self._set_field('last_change', time.time()) # toggle refresh module tracker list/set trigger_trackers_refresh(tracker_type) return self.uuid def edit(self, tracker_type, to_track, level, description=None, filters={}, tags=[], mails=[], webhook=None): # edit tracker old_type = self.get_type() old_to_track = self.get_tracked() old_level = self.get_level() user_id = self.get_user() # YARA if tracker_type == 'yara_custom' or tracker_type == 'yara_default': # create yara rule if tracker_type == 'yara_default' and old_type == 'yara': if not is_default_yara_rule(old_to_track): filepath = get_yara_rule_file_by_tracker_name(old_to_track) if filepath: os.remove(filepath) to_track = save_yara_rule(tracker_type, to_track, tracker_uuid=self.uuid) tracker_type = 'yara' # TODO TYPO EDIT elif tracker_type == 'typosquatting': pass if tracker_type != old_type: # LEVEL if old_level == 0: r_tracker.srem(f'user:tracker:{user_id}:{old_type}', self.uuid) elif old_level == 1: r_tracker.srem(f'global:tracker:{old_type}', self.uuid) self._set_level(level, tracker_type=tracker_type, user=user_id) # Delete OLD YARA Rule File if old_type == 'yara': if not is_default_yara_rule(old_to_track): filepath = get_yara_rule_file_by_tracker_name(old_to_track) if filepath: os.remove(filepath) if old_type == 'typosquatting': r_tracker.delete(f'tracker:typosquatting:{old_to_track}') self._set_field('type', tracker_type) # create all tracker set r_tracker.srem(f'all:tracker:{old_type}', old_to_track) r_tracker.sadd(f'all:tracker:{tracker_type}', to_track) # create tracker - uuid map r_tracker.srem(f'all:tracker_uuid:{old_type}:{old_to_track}', self.uuid) r_tracker.sadd(f'all:tracker_uuid:{tracker_type}:{to_track}', self.uuid) # create all tracker set by type r_tracker.srem(f'trackers:all:{old_type}', self.uuid) r_tracker.sadd(f'trackers:all:{tracker_type}', self.uuid) # Same Type elif level != old_level: if level == 0: r_tracker.srem('global:tracker', self.uuid) elif level == 1: r_tracker.srem(f'user:tracker:{user_id}', self.uuid) self._set_level(level, tracker_type=tracker_type, user=user_id) # To Track Edited if to_track != old_to_track: self._set_field('tracked', to_track) self._set_field('description', description) self._set_field('webhook', webhook) # Tags nb_old_tags = r_tracker.scard(f'tracker:tags:{self.uuid}') if nb_old_tags > 0 or tags: self._del_tags() self._set_tags(tags) # Mails nb_old_mails = r_tracker.scard(f'tracker:mail:{self.uuid}') if nb_old_mails > 0 or mails: self._del_mails() self._set_mails(mails) # Filters self.del_filters(old_type, old_to_track) if not filters: filters = {} for obj_type in get_objects_tracked(): filters[obj_type] = {} else: self.set_filters(filters) for obj_type in filters: r_tracker.sadd(f'trackers:objs:{tracker_type}:{obj_type}', to_track) r_tracker.sadd(f'trackers:uuid:{tracker_type}:{to_track}', f'{self.uuid}:{obj_type}') # Refresh Trackers trigger_trackers_refresh(tracker_type) if tracker_type != old_type: trigger_trackers_refresh(old_type) self._set_field('last_change', time.time()) return self.uuid def delete(self): for obj in self.get_objs(): self.remove(obj[0], obj[1], obj[2]) tracker_type = self.get_type() tracked = self.get_tracked() r_tracker.srem(f'all:tracker:{tracker_type}', tracked) # tracker - uuid map r_tracker.srem(f'all:tracker_uuid:{tracker_type}:{tracked}', self.uuid) r_tracker.srem('trackers:all', self.uuid) r_tracker.srem(f'trackers:all:{tracker_type}', self.uuid) if tracker_type == 'typosquatting': r_tracker.delete(f'tracker:typosquatting:{tracked}') elif tracker_type == 'yara': if not is_default_yara_rule(tracked): filepath = get_yara_rule_file_by_tracker_name(tracked) if filepath: os.remove(filepath) # Filters filters = get_objects_tracked() for obj_type in filters: r_tracker.srem(f'trackers:objs:{tracker_type}:{obj_type}', tracked) r_tracker.srem(f'trackers:uuid:{tracker_type}:{tracked}', f'{self.uuid}:{obj_type}') self._del_mails() self._del_tags() level = self.get_level() if level == 0: # user only user = self.get_user() r_tracker.srem(f'user:tracker:{user}', self.uuid) r_tracker.srem(f'user:tracker:{user}:{tracker_type}', self.uuid) elif level == 1: # global r_tracker.srem('global:tracker', self.uuid) r_tracker.srem(f'global:tracker:{tracker_type}', self.uuid) # meta r_tracker.delete(f'tracker:{self.uuid}') trigger_trackers_refresh(tracker_type) def create_tracker(tracker_type, to_track, user_id, level, description=None, filters={}, tags=[], mails=[], webhook=None, tracker_uuid=None): if not tracker_uuid: tracker_uuid = str(uuid.uuid4()) tracker = Tracker(tracker_uuid) return tracker.create(tracker_type, to_track, user_id, level, description=description, filters=filters, tags=tags, mails=mails, webhook=webhook) def _re_create_tracker(tracker_type, tracker_uuid, to_track, user_id, level, description=None, filters={}, tags=[], mails=[], webhook=None, first_seen=None, last_seen=None): create_tracker(tracker_type, to_track, user_id, level, description=description, filters=filters, tags=tags, mails=mails, webhook=webhook, tracker_uuid=tracker_uuid) def get_trackers_types(): return ['word', 'set', 'regex', 'typosquatting', 'yara'] def get_trackers(): return r_tracker.smembers(f'trackers:all') def get_trackers_by_type(tracker_type): return r_tracker.smembers(f'trackers:all:{tracker_type}') def _get_tracked_by_obj_type(tracker_type, obj_type): return r_tracker.smembers(f'trackers:objs:{tracker_type}:{obj_type}') def get_trackers_by_tracked_obj_type(tracker_type, obj_type, tracked): trackers_uuid = set() for res in r_tracker.smembers(f'trackers:uuid:{tracker_type}:{tracked}'): tracker_uuid, tracker_obj_type = res.split(':', 1) if tracker_obj_type == obj_type: trackers_uuid.add(tracker_uuid) return trackers_uuid def get_trackers_by_tracked(tracker_type, tracked): return r_tracker.smembers(f'all:tracker_uuid:{tracker_type}:{tracked}') def get_user_trackers_by_tracked(tracker_type, tracked, user_id): user_trackers = get_user_trackers(user_id, tracker_type=tracker_type) trackers_uuid = get_trackers_by_tracked(tracker_type, tracked) return trackers_uuid.intersection(user_trackers) def get_trackers_tracked_by_type(tracker_type): return r_tracker.smembers(f'all:tracker:{tracker_type}') def get_global_trackers(tracker_type=None): if tracker_type: return r_tracker.smembers(f'global:tracker:{tracker_type}') else: return r_tracker.smembers('global:tracker') def get_user_trackers(user_id, tracker_type=None): if tracker_type: return r_tracker.smembers(f'user:tracker:{user_id}:{tracker_type}') else: return r_tracker.smembers(f'user:tracker:{user_id}') def get_nb_global_trackers(tracker_type=None): if tracker_type: return r_tracker.scard(f'global:tracker:{tracker_type}') else: return r_tracker.scard('global:tracker') def get_nb_user_trackers(user_id, tracker_type=None): if tracker_type: return r_tracker.scard(f'user:tracker:{user_id}:{tracker_type}') else: return r_tracker.scard(f'user:tracker:{user_id}') def get_user_trackers_meta(user_id, tracker_type=None): metas = [] for tracker_uuid in get_user_trackers(user_id, tracker_type=tracker_type): tracker = Tracker(tracker_uuid) metas.append(tracker.get_meta(options={'description', 'mails', 'sparkline', 'tags'})) return metas def get_global_trackers_meta(tracker_type=None): metas = [] for tracker_uuid in get_global_trackers(tracker_type=tracker_type): tracker = Tracker(tracker_uuid) metas.append(tracker.get_meta(options={'description', 'mails', 'sparkline', 'tags'})) return metas def get_users_trackers_meta(): trackers = [] for tracker_uuid in get_trackers(): tracker = Tracker(tracker_uuid) if tracker.is_level_user(): trackers.append(tracker.get_meta(options={'mails', 'sparkline', 'tags'})) return trackers def get_trackers_graph_by_day(l_trackers, num_day=31, date_from=None, date_to=None): if date_from and date_to: date_range = Date.substract_date(date_from, date_to) else: date_range = Date.get_date_range(num_day) list_tracker_stats = [] for tracker_uuid in l_trackers: dict_tracker_data = [] tracker = Tracker(tracker_uuid) for date_day in date_range: nb_seen_this_day = tracker.get_nb_objs_by_date(date_day) if nb_seen_this_day is None: nb_seen_this_day = 0 dict_tracker_data.append({"date": date_day, "value": int(nb_seen_this_day)}) list_tracker_stats.append({"name": tracker.get_tracked(), "Data": dict_tracker_data}) return list_tracker_stats def get_trackers_dashboard(): trackers = [] for raw in r_tracker.lrange('trackers:dashboard', 0, -1): tracker_uuid, timestamp, obj_type, subtype, obj_id = raw.split(':', 4) tracker = Tracker(tracker_uuid) meta = tracker.get_meta(options={'tags'}) if not meta.get('type'): meta['type'] = 'Tracker DELETED' timestamp = datetime.datetime.fromtimestamp(float(timestamp)).strftime('%Y-%m-%d %H:%M:%S') meta['timestamp'] = timestamp trackers.append(meta) return trackers def get_user_dashboard(user_id): # TODO SORT + REMOVE OLDER ROWS (trim) trackers = [] for raw in r_tracker.lrange(f'trackers:user:{user_id}', 0, -1): tracker_uuid, timestamp, obj_type, subtype, obj_id = raw.split(':', 4) tracker = Tracker(tracker_uuid) meta = tracker.get_meta(options={'tags'}) timestamp = datetime.datetime.fromtimestamp(float(timestamp)).strftime('%Y-%m-%d %H:%M:%S') meta['timestamp'] = timestamp trackers.append(meta) return trackers def get_trackers_stats(user_id): stats = {'all': 0} for tracker_type in get_trackers_types(): nb_global = get_nb_global_trackers(tracker_type=tracker_type) nb_user = get_nb_user_trackers(user_id, tracker_type=tracker_type) stats[tracker_type] = nb_global + nb_user stats['all'] += nb_global + nb_user return stats ## Cache ## # TODO API: Check Tracker type def trigger_trackers_refresh(tracker_type): r_cache.set(f'tracker:refresh:{tracker_type}', time.time()) def get_tracker_last_updated_by_type(tracker_type): epoch_update = r_cache.get(f'tracker:refresh:{tracker_type}') if not epoch_update: epoch_update = 0 return float(epoch_update) # - Cache - # ## Objects ## def is_obj_tracked(obj_type, subtype, obj_id): return r_tracker.exists(f'obj:trackers:{obj_type}:{subtype}:{obj_id}') def get_obj_trackers(obj_type, subtype, obj_id): return r_tracker.smembers(f'obj:trackers:{obj_type}:{subtype}:{obj_id}') def delete_obj_trackers(obj_type, subtype, obj_id): for tracker_uuid in get_obj_trackers(obj_type, subtype, obj_id): tracker = Tracker(tracker_uuid) tracker.remove(obj_type, subtype, obj_id) ###################### #### TRACKERS ACL #### ## LEVEL ## def is_tracked_in_global_level(tracked, tracker_type): for tracker_uuid in get_trackers_by_tracked(tracker_type, tracked): tracker = Tracker(tracker_uuid) if tracker.is_level_global(): return True return False def is_tracked_in_user_level(tracked, tracker_type, user_id): trackers_uuid = get_user_trackers_by_tracked(tracker_type, tracked, user_id) if trackers_uuid: return True else: return False ## API ## def api_check_tracker_uuid(tracker_uuid): if not is_valid_uuid_v4(tracker_uuid): return {"status": "error", "reason": "Invalid uuid"}, 400 if not r_tracker.exists(f'tracker:{tracker_uuid}'): return {"status": "error", "reason": "Unknown uuid"}, 404 return None def api_check_tracker_acl(tracker_uuid, user_id): res = api_check_tracker_uuid(tracker_uuid) if res: return res tracker = Tracker(tracker_uuid) if tracker.is_level_user(): if tracker.get_user() != user_id or not User(user_id).is_in_role('admin'): return {"status": "error", "reason": "Access Denied"}, 403 return None def api_is_allowed_to_edit_tracker(tracker_uuid, user_id): if not is_valid_uuid_v4(tracker_uuid): return {"status": "error", "reason": "Invalid uuid"}, 400 tracker_creator = r_tracker.hget('tracker:{}'.format(tracker_uuid), 'user_id') if not tracker_creator: return {"status": "error", "reason": "Unknown uuid"}, 404 user = User(user_id) if not user.is_in_role('admin') and user_id != tracker_creator: return {"status": "error", "reason": "Access Denied"}, 403 return {"uuid": tracker_uuid}, 200 ##-- ACL --## #### FIX DB #### TODO ################################################################### def fix_tracker_stats_per_day(tracker_uuid): tracker = Tracker(tracker_uuid) date_from = tracker.get_date() date_to = Date.get_today_date_str() # delete stats r_tracker.delete(f'tracker:stat:{tracker_uuid}') r_tracker.hdel(f'tracker:{tracker_uuid}', 'first_seen') r_tracker.hdel(f'tracker:{tracker_uuid}', 'last_seen') # create new stats for date_day in Date.substract_date(date_from, date_to): date_day = int(date_day) nb_items = r_tracker.scard(f'tracker:item:{tracker_uuid}:{date_day}') if nb_items: r_tracker.zincrby(f'tracker:stat:{tracker_uuid}', nb_items, int(date_day)) # update first_seen/last_seen tracker.update_daterange(date_day) def fix_tracker_item_link(tracker_uuid): tracker = Tracker(tracker_uuid) date_from = tracker.get_first_seen() date_to = tracker.get_last_seen() if date_from and date_to: for date_day in Date.substract_date(date_from, date_to): l_items = r_tracker.smembers(f'tracker:item:{tracker_uuid}:{date_day}') for item_id in l_items: r_tracker.sadd(f'obj:trackers:item:{item_id}', tracker_uuid) def fix_all_tracker_uuid_list(): r_tracker.delete(f'trackers:all') for tracker_type in get_trackers_types(): r_tracker.delete(f'trackers:all:{tracker_type}') for tracked in get_trackers_tracked_by_type(tracker_type): l_tracker_uuid = get_trackers_by_tracked(tracker_type, tracked) for tracker_uuid in l_tracker_uuid: r_tracker.sadd(f'trackers:all', tracker_uuid) r_tracker.sadd(f'trackers:all:{tracker_type}', tracker_uuid) ##-- FIX DB --## #### CREATE TRACKER #### def api_validate_tracker_to_add(to_track, tracker_type, nb_words=1): if tracker_type == 'regex': if not is_valid_regex(to_track): return {"status": "error", "reason": "Invalid regex"}, 400 elif tracker_type == 'word' or tracker_type == 'set': # force lowercase to_track = to_track.lower() word_set = set(to_track) set_inter = word_set.intersection(special_characters) if set_inter: return {"status": "error", "reason": f'special character(s) not allowed: {set_inter}', "message": "Please use a python regex or remove all special characters"}, 400 words = to_track.split() # not a word if tracker_type == 'word' and len(words) > 1: tracker_type = 'set' # output format: tracker1,tracker2,tracker3;2 if tracker_type == 'set': try: nb_words = int(nb_words) except TypeError: nb_words = 1 if nb_words == 0: nb_words = 1 words_set = set(words) words_set = sorted(words_set) if nb_words > len(words_set): nb_words = len(words_set) to_track = ",".join(words_set) to_track = f"{to_track};{nb_words}" elif tracker_type == 'typosquatting': to_track = to_track.lower() # Take only the first term domain = to_track.split(" ") if len(domain) > 1: return {"status": "error", "reason": "Only one domain is accepted at a time"}, 400 if "." not in to_track: return {"status": "error", "reason": "Invalid domain name"}, 400 elif tracker_type == 'yara_custom': if not is_valid_yara_rule(to_track): return {"status": "error", "reason": "Invalid custom Yara Rule"}, 400 elif tracker_type == 'yara_default': if not is_valid_default_yara_rule(to_track): return {"status": "error", "reason": "The Yara Rule doesn't exist"}, 400 else: return {"status": "error", "reason": "Incorrect type"}, 400 return {"status": "success", "tracked": to_track, "type": tracker_type}, 200 def api_add_tracker(dict_input, user_id): to_track = dict_input.get('tracked', None) if not to_track: return {"status": "error", "reason": "Tracker not provided"}, 400 tracker_type = dict_input.get('type', None) if not tracker_type: return {"status": "error", "reason": "Tracker type not provided"}, 400 nb_words = dict_input.get('nb_words', 1) description = dict_input.get('description', '') description = escape(description) webhook = dict_input.get('webhook', '') webhook = escape(webhook) res = api_validate_tracker_to_add(to_track, tracker_type, nb_words=nb_words) if res[1] != 200: return res to_track = res[0]['tracked'] tracker_type = res[0]['type'] tags = dict_input.get('tags', []) mails = dict_input.get('mails', []) res = verify_mail_list(mails) if res: return res # Filters # TODO MOVE ME filters = dict_input.get('filters', {}) if filters: if filters.keys() == {'decoded', 'item', 'pgp', 'title'} and set(filters['pgp'].get('subtypes', [])) == {'mail', 'name'}: filters = {} for obj_type in filters: if obj_type not in get_objects_tracked(): return {"status": "error", "reason": "Invalid Tracker Object type"}, 400 if obj_type == 'pgp': if set(filters['pgp'].get('subtypes', [])) == {'mail', 'name'}: filters['pgp'].pop('subtypes') for filter_name in filters[obj_type]: if filter_name not in {'mimetypes', 'sources', 'subtypes'}: return {"status": "error", "reason": "Invalid Filter"}, 400 elif filter_name == 'mimetypes': # TODO pass elif filter_name == 'sources': if obj_type == 'item': res = item_basic.verify_sources_list(filters['item']['sources']) if res: return res else: return {"status": "error", "reason": "Invalid Filter sources"}, 400 elif filter_name == 'subtypes': obj_subtypes = set(get_object_all_subtypes(obj_type)) for subtype in filters[obj_type]['subtypes']: if subtype not in obj_subtypes: return {"status": "error", "reason": "Invalid Tracker Object subtype"}, 400 level = dict_input.get('level', 1) try: level = int(level) except TypeError: level = 1 if level not in range(0, 1): level = 1 tracker_uuid = create_tracker(tracker_type, to_track, user_id, level, description=description, filters=filters, tags=tags, mails=mails, webhook=webhook) return {'tracked': to_track, 'type': tracker_type, 'uuid': tracker_uuid}, 200 def api_edit_tracker(dict_input, user_id): tracker_uuid = dict_input.get('uuid') res = api_check_tracker_acl(tracker_uuid, user_id) if res: return res tracker = Tracker(tracker_uuid) to_track = dict_input.get('tracked', None) if not to_track: return {"status": "error", "reason": "Tracker not provided"}, 400 tracker_type = dict_input.get('type', None) if not tracker_type: return {"status": "error", "reason": "Tracker type not provided"}, 400 nb_words = dict_input.get('nb_words', 1) description = dict_input.get('description', '') description = escape(description) webhook = dict_input.get('webhook', '') webhook = escape(webhook) res = api_validate_tracker_to_add(to_track, tracker_type, nb_words=nb_words) if res[1] != 200: return res to_track = res[0]['tracked'] tracker_type = res[0]['type'] tags = dict_input.get('tags', []) mails = dict_input.get('mails', []) res = verify_mail_list(mails) if res: return res # Filters # TODO MOVE ME filters = dict_input.get('filters', {}) if filters: if filters.keys() == {'decoded', 'item', 'pgp', 'title'} and set(filters['pgp'].get('subtypes', [])) == {'mail', 'name'}: if not filters['decoded'] and not filters['item']: filters = {} for obj_type in filters: if obj_type not in get_objects_tracked(): return {"status": "error", "reason": "Invalid Tracker Object type"}, 400 if obj_type == 'pgp': if set(filters['pgp'].get('subtypes', [])) == {'mail', 'name'}: filters['pgp'].pop('subtypes') for filter_name in filters[obj_type]: if filter_name not in {'mimetypes', 'sources', 'subtypes'}: return {"status": "error", "reason": "Invalid Filter"}, 400 elif filter_name == 'mimetypes': # TODO pass elif filter_name == 'sources': if obj_type == 'item': res = item_basic.verify_sources_list(filters['item']['sources']) if res: return res else: return {"status": "error", "reason": "Invalid Filter sources"}, 400 elif filter_name == 'subtypes': obj_subtypes = set(get_object_all_subtypes(obj_type)) for subtype in filters[obj_type]['subtypes']: if subtype not in obj_subtypes: return {"status": "error", "reason": "Invalid Tracker Object subtype"}, 400 level = dict_input.get('level', 1) try: level = int(level) except TypeError: level = 1 if level not in range(0, 1): level = 1 tracker.edit(tracker_type, to_track, level, description=description, filters=filters, tags=tags, mails=mails, webhook=webhook) return {'tracked': to_track, 'type': tracker_type, 'uuid': tracker_uuid}, 200 def api_delete_tracker(data, user_id): tracker_uuid = data.get('uuid') res = api_check_tracker_acl(tracker_uuid, user_id) if res: return res tracker = Tracker(tracker_uuid) return tracker.delete(), 200 ## -- CREATE TRACKER -- ## #################### #### WORD - SET #### def get_tracked_words(): to_track = {} for obj_type in get_objects_tracked(): to_track[obj_type] = _get_tracked_by_obj_type('word', obj_type) return to_track def get_tracked_sets(): to_track = {} for obj_type in get_objects_tracked(): to_track[obj_type] = [] for tracked in _get_tracked_by_obj_type('set', obj_type): res = tracked.split(';') nb_words = int(res[1]) words_set = res[0].split(',') to_track[obj_type].append({'words': words_set, 'nb': nb_words, 'tracked': tracked}) return to_track def get_text_word_frequency(content, filtering=True): content = content.lower() words_dict = defaultdict(int) if filtering: blob = TextBlob(content, tokenizer=tokenizer) else: blob = TextBlob(content) for word in blob.tokens: words_dict[word] += 1 return words_dict ############### #### REGEX #### def get_tracked_regexs(): to_track = {} for obj_type in get_objects_tracked(): to_track[obj_type] = [] for tracked in _get_tracked_by_obj_type('regex', obj_type): to_track[obj_type].append({'regex': re.compile(tracked), 'tracked': tracked}) return to_track ######################## #### TYPO SQUATTING #### def get_tracked_typosquatting_domains(tracked): return r_tracker.smembers(f'tracker:typosquatting:{tracked}') def get_tracked_typosquatting(): to_track = {} for obj_type in get_objects_tracked(): to_track[obj_type] = [] for tracked in _get_tracked_by_obj_type('typosquatting', obj_type): to_track[obj_type].append({'domains': get_tracked_typosquatting_domains(tracked), 'tracked': tracked}) return to_track ############## #### YARA #### def get_yara_rules_dir(): return os.path.join(os.environ['AIL_BIN'], 'trackers', 'yara') def get_yara_rules_default_dir(): return os.path.join(os.environ['AIL_BIN'], 'trackers', 'yara', 'ail-yara-rules', 'rules') # # TODO: cache + update def get_all_default_yara_rules_types(): yara_dir = get_yara_rules_default_dir() all_yara_types = next(os.walk(yara_dir))[1] # save in cache ? return all_yara_types # # TODO: cache + update def get_all_default_yara_files(): yara_dir = get_yara_rules_default_dir() all_default_yara_files = {} for rules_type in get_all_default_yara_rules_types(): all_default_yara_files[rules_type] = os.listdir(os.path.join(yara_dir, rules_type)) return all_default_yara_files def get_all_default_yara_rules_by_type(yara_types): all_default_yara_files = get_all_default_yara_files() if yara_types in all_default_yara_files: return all_default_yara_files[yara_types] else: return [] def get_all_tracked_yara_files(filter_disabled=False): yara_files = r_tracker.smembers('all:tracker:yara') if not yara_files: yara_files = [] if filter_disabled: pass return yara_files def get_tracked_yara_rules(): to_track = {} for obj_type in get_objects_tracked(): rules = {} for tracked in _get_tracked_by_obj_type('yara', obj_type): rule = os.path.join(get_yara_rules_dir(), tracked) if not os.path.exists(rule): logger.critical(f"Yara rule don't exists {tracked} : {obj_type}") else: rules[tracked] = rule to_track[obj_type] = yara.compile(filepaths=rules) print(to_track) return to_track def reload_yara_rules(): yara_files = get_all_tracked_yara_files() # {uuid: filename} rule_dict = {} for yar_path in yara_files: for tracker_uuid in get_trackers_by_tracked('yara', yar_path): rule_dict[tracker_uuid] = os.path.join(get_yara_rules_dir(), yar_path) for tracker_uuid in rule_dict: if not os.path.isfile(rule_dict[tracker_uuid]): # TODO IGNORE + LOGS raise Exception(f"Error: {rule_dict[tracker_uuid]} doesn't exists") rules = yara.compile(filepaths=rule_dict) return rules def is_valid_yara_rule(yara_rule): try: yara.compile(source=yara_rule) return True except: return False def is_default_yara_rule(tracked_yara_name): yara_dir = get_yara_rules_dir() filename = os.path.join(yara_dir, tracked_yara_name) filename = os.path.realpath(filename) try: if tracked_yara_name.split('/')[0] == 'custom-rules': return False except: return False if not os.path.commonprefix([filename, yara_dir]) == yara_dir: return False else: if os.path.isfile(filename): return True return False def is_valid_default_yara_rule(yara_rule, verbose=True): yara_dir = get_yara_rules_default_dir() filename = os.path.join(yara_dir, yara_rule) filename = os.path.realpath(filename) # incorrect filename if not os.path.commonprefix([filename, yara_dir]) == yara_dir: if verbose: print('error: file transversal') print(yara_dir) print(filename) return False else: if os.path.isfile(filename): return True else: return False def save_yara_rule(yara_rule_type, yara_rule, tracker_uuid=None): if yara_rule_type == 'yara_custom': if not tracker_uuid: tracker_uuid = str(uuid.uuid4()) filename = os.path.join('custom-rules', tracker_uuid + '.yar') with open(os.path.join(get_yara_rules_dir(), filename), 'w') as f: f.write(str(yara_rule)) elif yara_rule_type == 'yara_default': filename = os.path.join('ail-yara-rules', 'rules', yara_rule) return filename def get_yara_rule_file_by_tracker_name(tracked_yara_name): yara_dir = get_yara_rules_dir() filename = os.path.join(yara_dir, tracked_yara_name) filename = os.path.realpath(filename) if not os.path.commonprefix([filename, yara_dir]) == yara_dir: print('error: file transversal') print(yara_dir) print(filename) return None return filename def get_yara_rule_content(yara_rule): yara_dir = get_yara_rules_dir() filename = os.path.join(yara_dir, yara_rule) filename = os.path.realpath(filename) # incorrect filename if not os.path.commonprefix([filename, yara_dir]) == yara_dir: return '' # # TODO: throw exception with open(filename, 'r') as f: rule_content = f.read() return rule_content def api_get_default_rule_content(default_yara_rule): yara_dir = get_yara_rules_default_dir() filename = os.path.join(yara_dir, default_yara_rule) filename = os.path.realpath(filename) if not os.path.commonprefix([filename, yara_dir]) == yara_dir: return {'status': 'error', 'reason': 'file traversal detected'}, 400 if not os.path.isfile(filename): return {'status': 'error', 'reason': 'yara rule not found'}, 400 with open(filename, 'r') as f: rule_content = f.read() return {'rule_name': default_yara_rule, 'content': rule_content}, 200 def get_yara_rule_content_restapi(request_dict): rule_name = request_dict.get('rule_name', None) if not request_dict: return {'status': 'error', 'reason': 'Malformed JSON'}, 400 if not rule_name: return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 yara_dir = get_yara_rules_dir() filename = os.path.join(yara_dir, rule_name) filename = os.path.realpath(filename) if not os.path.commonprefix([filename, yara_dir]) == yara_dir: return {'status': 'error', 'reason': 'File Path Traversal'}, 400 if not os.path.isfile(filename): return {'status': 'error', 'reason': 'yara rule not found'}, 400 with open(filename, 'r') as f: rule_content = f.read() rule_content = base64.b64encode((rule_content.encode('utf-8'))).decode('UTF-8') return {'status': 'success', 'content': rule_content}, 200 ## -- YARA -- ## ###################### #### RETRO - HUNT #### # state: pending/running/completed/paused class RetroHunt: def __init__(self, task_uuid): self.uuid = task_uuid def exists(self): return r_tracker.exists(f'retro_hunt:{self.uuid}') def _get_field(self, field): return r_tracker.hget(f'retro_hunt:{self.uuid}', field) def _set_field(self, field, value): return r_tracker.hset(f'retro_hunt:{self.uuid}', field, value) def get_creator(self): return self._get_field('creator') def get_date(self): return self._get_field('date') def get_last_analyzed(self): return self._get_field('last') def set_last_analyzed(self, obj_type, subtype, obj_id): return self._set_field('last', f'{obj_type}:{subtype}:{obj_id}') def get_last_analyzed_cache(self): r_cache.hget(f'retro_hunt:task:{self.uuid}', 'obj') def set_last_analyzed_cache(self, obj_type, subtype, obj_id): r_cache.hset(f'retro_hunt:task:{self.uuid}', 'obj', f'{obj_type}:{subtype}:{obj_id}') def get_name(self): return self._get_field('name') def get_description(self): return self._get_field('description') def get_timeout(self): res = self._get_field('timeout') if res: return int(res) else: return 30 # # TODO: FIXME use instance limit def get_filters(self): filters = self._get_field('filters') if not filters: return {} else: return json.loads(filters) def set_filters(self, filters): if filters: self._set_field('filters', json.dumps(filters)) def get_tags(self): return r_tracker.smembers(f'retro_hunt:tags:{self.uuid}') def get_mails(self): return r_tracker.smembers(f'retro_hunt:mails:{self.uuid}') def get_state(self): return self._get_field('state') def _set_state(self, new_state): curr_state = self.get_state() if curr_state: r_tracker.srem(f'retro_hunt:task:{curr_state}', self.uuid) r_tracker.sadd(f'retro_hunts:{new_state}', self.uuid) self._set_field('state', new_state) def get_rule(self, r_compile=False): rule = self._get_field('rule') if r_compile: rule = os.path.join(get_yara_rules_dir(), rule) rule_dict = {self.uuid: os.path.join(get_yara_rules_dir(), rule)} rule = yara.compile(filepaths=rule_dict) return rule # add timeout ? def get_meta(self, options=set()): meta = {'uuid': self.uuid, 'name': self.get_name(), 'state': self.get_state(), 'rule': self.get_rule(), } if 'creator' in options: meta['creator'] = self.get_creator() if 'date' in options: meta['date'] = self.get_date() if 'description' in options: meta['description'] = self.get_description() if 'mails' in options: meta['mails'] = self.get_mails() if 'nb_match' in options: meta['nb_match'] = self.get_nb_match() if 'nb_objs' in options: meta['nb_objs'] = self.get_nb_objs() if 'progress' in options: meta['progress'] = self.get_progress() if 'filters' in options: meta['filters'] = self.get_filters() if 'tags' in options: meta['tags'] = self.get_tags() return meta def is_paused(self): return r_tracker.sismember('retro_hunts:paused', self.uuid) def to_pause(self): to_pause = r_cache.hget(f'retro_hunt:{self.uuid}', 'pause') if to_pause: return True else: return False def pause(self): self._set_state('paused') r_cache.hset(f'retro_hunt:{self.uuid}', 'pause', time.time()) self.clear_cache() def resume(self): r_cache.hdel(f'retro_hunt:{self.uuid}', 'pause') self._set_state('pending') def is_running(self): return r_tracker.sismember('retro_hunts:running', self.uuid) def run(self): # TODO ADD MORE CHECK self._set_state('running') def complete(self): self._set_state('completed') self.clear_cache() r_tracker.hdel(f'retro_hunt:{self.uuid}', 'last') def get_progress(self): if self.get_state() == 'completed': progress = 100 else: progress = r_cache.hget(f'retro_hunt:{self.uuid}', 'progress') if not progress: progress = self._get_field('progress') return progress def set_progress(self, progress): res = r_cache.hset(f'retro_hunt:{self.uuid}', 'progress', progress) if res: self._set_field('progress', progress) def get_nb_match(self): return self._get_field('nb_match') def _incr_nb_match(self): r_tracker.hincrby(f'retro_hunt:{self.uuid}', 'nb_match', 1) def _decr_nb_match(self): r_tracker.hincrby(f'retro_hunt:{self.uuid}', 'nb_match', -1) def _set_nb_match(self, nb_match): self._set_field('nb_match', nb_match) def clear_cache(self): r_cache.delete(f'retro_hunt:{self.uuid}') def get_nb_objs_by_type(self, obj_type): return r_tracker.scard(f'retro_hunt:objs:{self.uuid}:{obj_type}') def get_objs_by_type(self, obj_type): return r_tracker.smembers(f'retro_hunt:objs:{self.uuid}:{obj_type}') def get_nb_objs(self): objs = {} for obj_type in get_objects_retro_hunted(): nb = self.get_nb_objs_by_type(obj_type) if nb: objs[obj_type] = nb return objs def get_objs(self): objs = [] for obj_type in get_objects_retro_hunted(): for obj in self.get_objs_by_type(obj_type): subtype, obj_id = obj.split(':', 1) objs.append((obj_type, subtype, obj_id)) return objs def add(self, obj_type, subtype, obj_id): # match by object type: r_tracker.sadd(f'retro_hunt:objs:{self.uuid}:{obj_type}', f'{subtype}:{obj_id}') # MAP object -> retro hunt r_tracker.sadd(f'obj:retro_hunts:{obj_type}:{subtype}:{obj_id}', self.uuid) self._incr_nb_match() def remove(self, obj_type, subtype, obj_id): # match by object type: r_tracker.srem(f'retro_hunt:objs:{self.uuid}:{obj_type}', f'{subtype}:{obj_id}') # MAP object -> retro hunt r_tracker.srem(f'obj:retro_hunts:{obj_type}:{subtype}:{obj_id}', self.uuid) self._decr_nb_match() def create(self, name, rule, creator, description=None, filters=[], mails=[], tags=[], timeout=30, state='pending'): if self.exists(): raise Exception('Error: Retro Hunt Task already exists') self._set_field('name', escape(name)) self._set_field('rule', rule) self._set_field('date', datetime.date.today().strftime("%Y%m%d")) self._set_field('name', escape(name)) self._set_field('creator', creator) if description: self._set_field('description', description) if timeout: self._set_field('timeout', int(timeout)) for tag in tags: # tag = escape(tag) r_tracker.sadd(f'retro_hunt:tags:{self.uuid}', tag) Tag.create_custom_tag(tag) for mail in mails: r_tracker.sadd(f'retro_hunt:mails:{self.uuid}', escape(mail)) if filters: self.set_filters(filters) r_tracker.sadd('retro_hunts:all', self.uuid) # add to pending tasks if state not in ('pending', 'completed', 'paused'): state = 'pending' self._set_state(state) def delete(self): if self.is_running() and self.get_state() != 'completed': return None # Delete custom rule rule = self.get_rule() if not is_default_yara_rule(rule): filepath = get_yara_rule_file_by_tracker_name(rule) if filepath: os.remove(filepath) r_tracker.srem('retro_hunts:pending', self.uuid) r_tracker.delete(f'retro_hunts:{self.uuid}') r_tracker.delete(f'retro_hunt:tags:{self.uuid}') r_tracker.delete(f'retro_hunt:mails:{self.uuid}') for obj in self.get_objs(): self.remove(obj[0], obj[1], obj[2]) r_tracker.srem('retro_hunts:all', self.uuid) r_tracker.srem('retro_hunts:pending', self.uuid) r_tracker.srem('retro_hunts:paused', self.uuid) r_tracker.srem('retro_hunts:completed', self.uuid) self.clear_cache() return self.uuid def create_retro_hunt(name, rule_type, rule, creator, description=None, filters=[], mails=[], tags=[], timeout=30, state='pending', task_uuid=None): if not task_uuid: task_uuid = str(uuid.uuid4()) retro_hunt = RetroHunt(task_uuid) # rule_type: yara_default - yara custom rule = save_yara_rule(rule_type, rule, tracker_uuid=retro_hunt.uuid) retro_hunt.create(name, rule, creator, description=description, mails=mails, tags=tags, timeout=timeout, filters=filters, state=state) return retro_hunt.uuid # TODO # def _re_create_retro_hunt_task(name, rule, date, date_from, date_to, creator, sources, tags, mails, timeout, description, task_uuid, state='pending', nb_match=0, last_id=None): # retro_hunt = RetroHunt(task_uuid) # retro_hunt.create(name, rule, date_from, date_to, creator, description=description, mails=mails, tags=tags, # timeout=timeout, sources=sources, state=state) # if last_id: # set_retro_hunt_last_analyzed(task_uuid, last_id) # retro_hunt._set_nb_match(nb_match) # retro_hunt._set_field('date', date) ## ? ? ? # set tags # set mails # limit mail # SET Retro Hunts def get_all_retro_hunt_tasks(): return r_tracker.smembers('retro_hunts:all') def get_retro_hunt_pending_tasks(): return r_tracker.smembers('retro_hunts:pending') def get_retro_hunt_running_tasks(): return r_tracker.smembers('retro_hunts:running') def get_retro_hunt_paused_tasks(): return r_tracker.smembers('retro_hunts:paused') def get_retro_hunt_completed_tasks(): return r_tracker.smembers('retro_hunts:completed') ## Change STATES ## def get_retro_hunt_task_to_start(): task_uuid = r_tracker.spop('retro_hunts:pending') if task_uuid: retro_hunt = RetroHunt(task_uuid) retro_hunt.run() return task_uuid ## Metadata ## def get_retro_hunt_metas(): tasks = [] for task_uuid in get_all_retro_hunt_tasks(): retro_hunt = RetroHunt(task_uuid) tasks.append(retro_hunt.get_meta(options={'date', 'progress', 'nb_match', 'tags'})) return tasks ## API ## def api_check_retro_hunt_task_uuid(task_uuid): if not is_valid_uuid_v4(task_uuid): return {"status": "error", "reason": "Invalid uuid"}, 400 retro_hunt = RetroHunt(task_uuid) if not retro_hunt.exists(): return {"status": "error", "reason": "Unknown uuid"}, 404 return None def api_pause_retro_hunt_task(task_uuid): res = api_check_retro_hunt_task_uuid(task_uuid) if res: return res retro_hunt = RetroHunt(task_uuid) task_state = retro_hunt.get_state() if task_state not in ['pending', 'running']: return {"status": "error", "reason": f"Task {task_uuid} not paused, current state: {task_state}"}, 400 retro_hunt.pause() return task_uuid, 200 def api_resume_retro_hunt_task(task_uuid): res = api_check_retro_hunt_task_uuid(task_uuid) if res: return res retro_hunt = RetroHunt(task_uuid) if not retro_hunt.is_paused(): return {"status": "error", "reason": f"Task {task_uuid} not paused, current state: {retro_hunt.get_state()}"}, 400 retro_hunt.resume() return task_uuid, 200 def api_validate_rule_to_add(rule, rule_type): if rule_type == 'yara_custom': if not is_valid_yara_rule(rule): return {"status": "error", "reason": "Invalid custom Yara Rule"}, 400 elif rule_type == 'yara_default': if not is_valid_default_yara_rule(rule): return {"status": "error", "reason": "The Yara Rule doesn't exist"}, 400 else: return {"status": "error", "reason": "Incorrect type"}, 400 return {"status": "success", "rule": rule, "type": rule_type}, 200 def api_create_retro_hunt_task(dict_input, creator): # # TODO: API: check mandatory arg # # TODO: TIMEOUT # timeout=30 rule = dict_input.get('rule', None) if not rule: return {"status": "error", "reason": "Retro Hunt Rule not provided"}, 400 task_type = dict_input.get('type', None) if not task_type: return {"status": "error", "reason": "type not provided"}, 400 # # TODO: limit name = dict_input.get('name', '') name = escape(name) name = name[:60] # # TODO: limit description = dict_input.get('description', '') description = escape(description) description = description[:1000] res = api_validate_rule_to_add(rule, task_type) if res[1] != 200: return res tags = dict_input.get('tags', []) # TODO escape custom tags mails = dict_input.get('mails', []) # TODO escape mails res = verify_mail_list(mails) if res: return res # Filters # TODO MOVE ME filters = dict_input.get('filters', {}) if filters: if filters.keys() == get_objects_retro_hunted(): filters = {} for obj_type in filters: if obj_type not in get_objects_retro_hunted(): return {"status": "error", "reason": "Invalid Tracker Object type"}, 400 for filter_name in filters[obj_type]: if filter_name not in {'date_from', 'date_to', 'mimetypes', 'sources', 'subtypes'}: return {"status": "error", "reason": "Invalid Filter"}, 400 elif filter_name == 'date_from': if not Date.validate_str_date(filters[obj_type]['date_from']): return {"status": "error", "reason": "Invalid date_from"}, 400 elif filter_name == 'date_to': if not Date.validate_str_date(filters[obj_type]['date_from']): return {"status": "error", "reason": "Invalid date_to"}, 400 elif filter_name == 'mimetypes': # TODO sanityze mimetypes pass elif filter_name == 'sources': if obj_type == 'item': res = item_basic.verify_sources_list(filters['item']['sources']) if res: return res else: return {"status": "error", "reason": "Invalid Filter sources"}, 400 elif filter_name == 'subtypes': obj_subtypes = set(get_object_all_subtypes(obj_type)) for subtype in filters[obj_type]['subtypes']: if subtype not in obj_subtypes: return {"status": "error", "reason": "Invalid Tracker Object subtype"}, 400 if 'date_from' and 'date_to' in filters: res = Date.api_validate_str_date_range(filters[obj_type]['date_from'], filters[obj_type]['date_to']) if res: return res task_uuid = create_retro_hunt(name, task_type, rule, creator, description=description, mails=mails, tags=tags, timeout=30, filters=filters) return {'name': name, 'rule': rule, 'type': task_type, 'uuid': task_uuid}, 200 def api_delete_retro_hunt_task(task_uuid): res = api_check_retro_hunt_task_uuid(task_uuid) if res: return res retro_hunt = RetroHunt(task_uuid) if retro_hunt.is_running() and retro_hunt.get_state() != 'completed': return {"status": "error", "reason": "You can't delete a running task"}, 400 else: return retro_hunt.delete(), 200 ################################################################################ ################################################################################ ################################################################################ ################################################################################ #### DB FIX #### TODO def _fix_db_custom_tags(): for tag in get_trackers_tags(): if not Tag.is_taxonomie_tag(tag) and not Tag.is_galaxy_tag(tag): Tag.create_custom_tag(tag) #### -- #### if __name__ == '__main__': _fix_db_custom_tags() # fix_all_tracker_uuid_list() # res = get_all_tracker_uuid() # print(len(res)) # import Term # Term.delete_term('5262ab6c-8784-4a55-b0ff-a471018414b4') #fix_tracker_stats_per_day('5262ab6c-8784-4a55-b0ff-a471018414b4') # tracker_uuid = '5262ab6c-8784-4a55-b0ff-a471018414b4' # fix_tracker_item_link(tracker_uuid) # res = get_item_all_trackers_uuid('archive/') # print(res) #res = is_valid_yara_rule('rule dummy { }') # res = create_tracker('test', 'word', 'admin@admin.test', 1, [], [], None, sources=['crawled', 'pastebin.com', 'rt/pastebin.com']) #res = create_tracker('circl\.lu', 'regex', 'admin@admin.test', 1, [], [], None, sources=['crawled','pastebin.com']) #print(res) #t_uuid = '1c2d35b0-9330-4feb-b454-da13007aa9f7' #res = get_tracker_sources('ail-yara-rules/rules/crypto/certificate.yar', 'yara') # sys.path.append(os.environ['AIL_BIN']) # from packages import Term # Term.delete_term('074ab4be-6049-45b5-a20e-8125a4e4f500') #res = get_items_to_analyze('archive/pastebin.com_pro/2020/05/15', last='archive/pastebin.com_pro/2020/05/15/zkHEgqjQ.gz') #get_retro_hunt_task_progress('0', nb_src_done=2) #res = set_cache_retro_hunt_task_progress('0', 100) #res = get_retro_hunt_task_nb_src_done('0', sources=['pastebin.com_pro', 'alerts/pastebin.com_pro', 'crawled']) #print(res) # sources = ['pastebin.com_pro', 'alerts/pastebin.com_pro', 'crawled'] # rule = 'custom-rules/4a8a3d04-f0b6-43ce-8e00-bdf47a8df241.yar' # name = 'retro_hunt_test_1' # description = 'circl retro hunt first test' # tags = ['retro_circl', 'circl'] # creator = 'admin@admin.test' # date_from = '20200610' # date_to = '20210630' #res = create_retro_hunt_task(name, rule, date_from, date_to, creator, sources=sources, tags=tags, description=description) #get_retro_hunt_nb_item_by_day(['80b402ef-a8a9-4e97-adb6-e090edcfd571'], date_from=None, date_to=None, num_day=31) #res = get_retro_hunt_nb_item_by_day(['c625f971-16e6-4331-82a7-b1e1b9efdec1'], date_from='20200610', date_to='20210630') #res = delete_retro_hunt_task('598687b6-f765-4f8b-861a-09ad76d0ab34') #print(res)