mirror of https://github.com/CIRCL/AIL-framework
1880 lines
68 KiB
Python
Executable File
1880 lines
68 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# -*-coding:UTF-8 -*
|
|
import json
|
|
import os
|
|
import logging
|
|
import logging.config
|
|
import re
|
|
import sys
|
|
import time
|
|
import uuid
|
|
import yara
|
|
import datetime
|
|
import base64
|
|
|
|
import math
|
|
|
|
from collections import defaultdict
|
|
from markupsafe import escape
|
|
from textblob import TextBlob
|
|
from nltk.tokenize import RegexpTokenizer
|
|
|
|
sys.path.append(os.environ['AIL_BIN'])
|
|
##################################
|
|
# Import Project packages
|
|
##################################
|
|
from packages import Date
|
|
from lib.ail_core import get_objects_tracked, get_object_all_subtypes, get_objects_retro_hunted
|
|
from lib import ail_logger
|
|
from lib import ConfigLoader
|
|
from lib import item_basic
|
|
from lib import Tag
|
|
from lib.Users import User
|
|
|
|
# LOGS
|
|
logging.config.dictConfig(ail_logger.get_config(name='modules'))
|
|
logger = logging.getLogger()
|
|
|
|
config_loader = ConfigLoader.ConfigLoader()
|
|
r_cache = config_loader.get_redis_conn("Redis_Cache")
|
|
r_tracker = config_loader.get_db_conn("Kvrocks_Trackers")
|
|
config_loader = None
|
|
|
|
# NLTK tokenizer
|
|
TOKENIZER = None
|
|
|
|
def init_tokenizer():
|
|
global TOKENIZER
|
|
TOKENIZER = RegexpTokenizer('[\&\~\:\;\,\.\(\)\{\}\|\[\]\\\\/\-/\=\'\"\%\$\?\@\+\#\_\^\<\>\!\*\n\r\t\s]+',
|
|
gaps=True, discard_empty=True)
|
|
|
|
def get_special_characters():
|
|
special_characters = set('[<>~!?@#$%^&*|()_-+={}":;,.\'\n\r\t]/\\')
|
|
special_characters.add('\\s')
|
|
return special_characters
|
|
|
|
###############
|
|
#### UTILS ####
|
|
def is_valid_uuid_v4(curr_uuid):
|
|
if not curr_uuid:
|
|
return False
|
|
curr_uuid = curr_uuid.replace('-', '')
|
|
try:
|
|
uuid_test = uuid.UUID(hex=curr_uuid, version=4)
|
|
return uuid_test.hex == curr_uuid
|
|
except:
|
|
return False
|
|
|
|
def is_valid_regex(tracker_regex):
|
|
try:
|
|
re.compile(tracker_regex)
|
|
return True
|
|
except:
|
|
return False
|
|
|
|
def is_valid_mail(email):
|
|
email_regex = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}'
|
|
email_regex = re.compile(email_regex)
|
|
result = email_regex.match(email)
|
|
if result:
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
def verify_mail_list(mail_list):
|
|
for mail in mail_list:
|
|
if not is_valid_mail(mail):
|
|
return {'status': 'error', 'reason': 'Invalid email', 'value': mail}, 400
|
|
return None
|
|
|
|
## -- UTILS -- ##
|
|
#################
|
|
|
|
###################
|
|
#### TRACKER ####
|
|
|
|
class Tracker:
|
|
def __init__(self, tracker_uuid):
|
|
self.uuid = tracker_uuid
|
|
|
|
def get_uuid(self):
|
|
return self.uuid
|
|
|
|
def exists(self):
|
|
return r_tracker.exists(f'tracker:{self.uuid}')
|
|
|
|
def _get_field(self, field):
|
|
return r_tracker.hget(f'tracker:{self.uuid}', field)
|
|
|
|
def _set_field(self, field, value):
|
|
r_tracker.hset(f'tracker:{self.uuid}', field, value)
|
|
|
|
def get_date(self):
|
|
return self._get_field('date')
|
|
|
|
def get_last_change(self, r_str=False):
|
|
last_change = self._get_field('last_change')
|
|
if r_str and last_change:
|
|
last_change = datetime.datetime.fromtimestamp(float(last_change)).strftime('%Y-%m-%d %H:%M:%S')
|
|
return last_change
|
|
|
|
def get_first_seen(self):
|
|
return self._get_field('first_seen')
|
|
|
|
def get_last_seen(self):
|
|
return self._get_field('last_seen')
|
|
|
|
def _set_first_seen(self, date):
|
|
self._set_field('first_seen', date)
|
|
|
|
def _set_last_seen(self, date):
|
|
self._set_field('last_seen', date)
|
|
|
|
def _exist_date(self, date):
|
|
return r_tracker.exists(f'tracker:objs:{self.uuid}:{date}')
|
|
|
|
# TODO: ADD CACHE ???
|
|
def update_daterange(self, date=None):
|
|
first_seen = self.get_first_seen()
|
|
# Added Object
|
|
if date:
|
|
date = int(date)
|
|
first_seen = self.get_first_seen()
|
|
# if op == 'add':
|
|
if not first_seen:
|
|
self._set_first_seen(date)
|
|
self._set_last_seen(date)
|
|
else:
|
|
first_seen = int(first_seen)
|
|
last_seen = int(self.get_last_seen())
|
|
if date < first_seen:
|
|
self._set_first_seen(date)
|
|
if date > last_seen:
|
|
self._set_last_seen(date)
|
|
else:
|
|
last_seen = self.get_last_seen()
|
|
if first_seen and last_seen:
|
|
valid_first_seen = self._exist_date(first_seen)
|
|
valid_last_seen = self._exist_date(last_seen)
|
|
# update first seen
|
|
if not valid_first_seen:
|
|
for date in Date.get_daterange(first_seen, last_seen):
|
|
if self._exist_date(date):
|
|
self._set_first_seen(date)
|
|
valid_first_seen = True
|
|
break
|
|
# update last seen
|
|
if not valid_last_seen:
|
|
for date in reversed(Date.get_daterange(first_seen, last_seen)):
|
|
if self._exist_date(date):
|
|
self._set_first_seen(date)
|
|
valid_last_seen = True
|
|
break
|
|
if not valid_first_seen or not valid_last_seen:
|
|
r_tracker.hdel(f'tracker:{self.uuid}', 'first_seen')
|
|
r_tracker.hdel(f'tracker:{self.uuid}', 'last_seen')
|
|
|
|
def get_description(self):
|
|
return self._get_field('description')
|
|
|
|
def get_level(self):
|
|
level = self._get_field('level')
|
|
if not level:
|
|
level = 0
|
|
return int(level)
|
|
|
|
def is_level_user(self):
|
|
return self.get_level() == 0
|
|
|
|
def is_level_global(self):
|
|
return self.get_level() == 1
|
|
|
|
def _set_level(self, level, tracker_type=None, user=None):
|
|
if not tracker_type:
|
|
tracker_type = self.get_type()
|
|
if level == 0: # user only
|
|
if not user:
|
|
user = self.get_user()
|
|
r_tracker.sadd(f'user:tracker:{user}', self.uuid)
|
|
r_tracker.sadd(f'user:tracker:{user}:{tracker_type}', self.uuid)
|
|
elif level == 1: # global
|
|
r_tracker.sadd('global:tracker', self.uuid)
|
|
r_tracker.sadd(f'global:tracker:{tracker_type}', self.uuid)
|
|
self._set_field('level', level)
|
|
|
|
def get_filters(self):
|
|
filters = self._get_field('filters')
|
|
if not filters:
|
|
return {}
|
|
else:
|
|
return json.loads(filters)
|
|
|
|
def set_filters(self, filters):
|
|
if filters:
|
|
self._set_field('filters', json.dumps(filters))
|
|
|
|
def del_filters(self, tracker_type, to_track):
|
|
filters = self.get_filters()
|
|
for obj_type in filters:
|
|
r_tracker.srem(f'trackers:objs:{tracker_type}:{obj_type}', to_track)
|
|
r_tracker.srem(f'trackers:uuid:{tracker_type}:{to_track}', f'{self.uuid}:{obj_type}')
|
|
r_tracker.hdel(f'tracker:{self.uuid}', 'filters')
|
|
|
|
def get_tracked(self):
|
|
return self._get_field('tracked')
|
|
|
|
def get_type(self):
|
|
return self._get_field('type')
|
|
|
|
def get_tags(self):
|
|
return r_tracker.smembers(f'tracker:tags:{self.uuid}')
|
|
|
|
def _set_tags(self, tags):
|
|
for tag in tags:
|
|
r_tracker.sadd(f'tracker:tags:{self.uuid}', tag)
|
|
Tag.create_custom_tag(tag) # TODO CUSTOM TAGS
|
|
|
|
def _del_tags(self):
|
|
return r_tracker.delete(f'tracker:tags:{self.uuid}')
|
|
|
|
def mail_export(self):
|
|
return r_tracker.exists(f'tracker:mail:{self.uuid}')
|
|
|
|
def get_mails(self):
|
|
return r_tracker.smembers(f'tracker:mail:{self.uuid}')
|
|
|
|
def _set_mails(self, mails):
|
|
for mail in mails:
|
|
r_tracker.sadd(f'tracker:mail:{self.uuid}', escape(mail))
|
|
|
|
def _del_mails(self):
|
|
r_tracker.delete(f'tracker:mail:{self.uuid}')
|
|
|
|
def get_user(self):
|
|
return self._get_field('user_id')
|
|
|
|
def webhook_export(self):
|
|
webhook = self.get_webhook()
|
|
return webhook is not None and webhook
|
|
|
|
def get_webhook(self):
|
|
return r_tracker.hget(f'tracker:{self.uuid}', 'webhook')
|
|
|
|
def get_sparkline(self, nb_day=6):
|
|
date_range_sparkline = Date.get_date_range(nb_day)
|
|
sparkline = []
|
|
for date in date_range_sparkline:
|
|
nb_seen_this_day = self.get_nb_objs_by_date(date)
|
|
if nb_seen_this_day is None:
|
|
nb_seen_this_day = 0
|
|
sparkline.append(int(nb_seen_this_day))
|
|
return sparkline
|
|
|
|
def get_rule(self):
|
|
yar_path = self.get_tracked()
|
|
return yara.compile(filepath=os.path.join(get_yara_rules_dir(), yar_path))
|
|
|
|
def get_meta(self, options):
|
|
if not options:
|
|
options = set()
|
|
meta = {'uuid': self.uuid,
|
|
'tracked': self.get_tracked(), # TODO TO CHECK
|
|
'type': self.get_type(),
|
|
'date': self.get_date(),
|
|
'first_seen': self.get_first_seen(),
|
|
'last_seen': self.get_last_seen()}
|
|
if 'user' in options:
|
|
meta['user'] = self.get_user()
|
|
if 'level' in options:
|
|
meta['level'] = self.get_level()
|
|
if 'description' in options:
|
|
meta['description'] = self.get_description()
|
|
if 'nb_objs' in options:
|
|
meta['nb_objs'] = self.get_nb_objs()
|
|
if 'tags' in options:
|
|
meta['tags'] = self.get_tags()
|
|
if 'filters' in options:
|
|
meta['filters'] = self.get_filters()
|
|
if 'mails' in options:
|
|
meta['mails'] = self.get_mails()
|
|
if 'webhooks' in options:
|
|
meta['webhook'] = self.get_webhook()
|
|
if 'sparkline' in options:
|
|
meta['sparkline'] = self.get_sparkline(6)
|
|
return meta
|
|
|
|
def _add_to_dashboard(self, obj_type, subtype, obj_id):
|
|
mess = f'{self.uuid}:{int(time.time())}:{obj_type}:{subtype}:{obj_id}'
|
|
if self.is_level_user():
|
|
user = self.get_user()
|
|
r_tracker.lpush(f'trackers:user:{user}', mess)
|
|
r_tracker.ltrim(f'trackers:user:{user}', 0, 9)
|
|
else:
|
|
r_tracker.lpush('trackers:dashboard', mess)
|
|
r_tracker.ltrim(f'trackers:dashboard', 0, 9)
|
|
|
|
def get_nb_objs_by_type(self, obj_type):
|
|
return r_tracker.scard(f'tracker:objs:{self.uuid}:{obj_type}')
|
|
|
|
def get_objs_by_type(self, obj_type):
|
|
return r_tracker.smembers(f'tracker:objs:{self.uuid}:{obj_type}')
|
|
|
|
def get_nb_objs(self):
|
|
objs = {}
|
|
for obj_type in get_objects_tracked():
|
|
nb = self.get_nb_objs_by_type(obj_type)
|
|
if nb:
|
|
objs[obj_type] = nb
|
|
return objs
|
|
|
|
def get_objs(self):
|
|
objs = []
|
|
for obj_type in get_objects_tracked():
|
|
for obj in self.get_objs_by_type(obj_type):
|
|
subtype, obj_id = obj.split(':', 1)
|
|
objs.append((obj_type, subtype, obj_id))
|
|
return objs
|
|
|
|
def get_nb_objs_by_date(self, date):
|
|
return r_tracker.scard(f'tracker:objs:{self.uuid}:{date}')
|
|
|
|
def get_objs_by_date(self, date):
|
|
return r_tracker.smembers(f'tracker:objs:{self.uuid}:{date}')
|
|
|
|
def get_objs_by_daterange(self, date_from, date_to):
|
|
objs = set()
|
|
for date in Date.get_daterange(date_from, date_to):
|
|
objs |= self.get_objs_by_date(date)
|
|
return objs
|
|
|
|
def get_obj_dates(self, obj_type, subtype, obj_id):
|
|
return r_tracker.smembers(f'obj:tracker:{obj_type}:{subtype}:{obj_id}:{self.uuid}')
|
|
|
|
# - TODO Data Retention TO Implement - #
|
|
# Or Daily/Monthly Global DB Cleanup:
|
|
# Iterate on each tracker:
|
|
# Iterate on each Obj:
|
|
# Iterate on each date:
|
|
# Delete from tracker range if date limit exceeded
|
|
# - TODO
|
|
def add(self, obj_type, subtype, obj_id, date=None):
|
|
if not subtype:
|
|
subtype = ''
|
|
if not date:
|
|
date = Date.get_today_date_str()
|
|
|
|
new_obj_date = r_tracker.sadd(f'tracker:objs:{self.uuid}:{date}', f'{obj_type}:{subtype}:{obj_id}')
|
|
r_tracker.sadd(f'obj:trackers:{obj_type}:{subtype}:{obj_id}', self.uuid)
|
|
|
|
# Only save object match date - Needed for the DB Cleaner
|
|
r_tracker.sadd(f'obj:tracker:{obj_type}:{subtype}:{obj_id}:{self.uuid}', date)
|
|
r_tracker.sadd(f'tracker:objs:{self.uuid}:{obj_type}', f'{subtype}:{obj_id}')
|
|
|
|
if new_obj_date:
|
|
self.update_daterange(date)
|
|
|
|
self._add_to_dashboard(obj_type, subtype, obj_id)
|
|
|
|
def remove(self, obj_type, subtype, obj_id):
|
|
if not subtype:
|
|
subtype = ''
|
|
|
|
for date in self.get_obj_dates(obj_type, subtype, obj_id):
|
|
r_tracker.srem(f'tracker:objs:{self.uuid}:{date}', f'{obj_type}:{subtype}:{obj_id}')
|
|
r_tracker.srem(f'obj:tracker:{obj_type}:{subtype}:{obj_id}:{self.uuid}', date)
|
|
|
|
r_tracker.srem(f'obj:trackers:{obj_type}:{subtype}:{obj_id}', self.uuid)
|
|
r_tracker.srem(f'tracker:objs:{self.uuid}:{obj_type}', f'{subtype}:{obj_id}')
|
|
self.update_daterange()
|
|
|
|
# TODO escape custom tags
|
|
# TODO escape mails ????
|
|
def create(self, tracker_type, to_track, user_id, level, description=None, filters={}, tags=[], mails=[], webhook=None):
|
|
if self.exists():
|
|
raise Exception('Error: Tracker already exists')
|
|
|
|
# YARA
|
|
if tracker_type == 'yara_custom' or tracker_type == 'yara_default':
|
|
to_track = save_yara_rule(tracker_type, to_track, tracker_uuid=self.uuid)
|
|
tracker_type = 'yara'
|
|
|
|
elif tracker_type == 'typosquatting':
|
|
|
|
from ail_typo_squatting import runAll
|
|
|
|
domain = to_track.split(" ")[0]
|
|
typo_generation = runAll(domain=domain, limit=math.inf, formatoutput="text", pathOutput="-", verbose=False) # TODO REPLACE LIMIT BY -1
|
|
for typo in typo_generation:
|
|
r_tracker.sadd(f'tracker:typosquatting:{to_track}', typo)
|
|
|
|
# create metadata
|
|
self._set_field('tracked', to_track)
|
|
self._set_field('type', tracker_type)
|
|
self._set_field('date', datetime.date.today().strftime("%Y%m%d"))
|
|
self._set_field('user_id', user_id)
|
|
if description:
|
|
self._set_field('description', escape(description))
|
|
if webhook:
|
|
self._set_field('webhook', webhook)
|
|
|
|
# create all tracker set
|
|
r_tracker.sadd(f'all:tracker:{tracker_type}', to_track)
|
|
# create tracker - uuid map
|
|
r_tracker.sadd(f'all:tracker_uuid:{tracker_type}:{to_track}', self.uuid)
|
|
r_tracker.sadd('trackers:all', self.uuid)
|
|
r_tracker.sadd(f'trackers:all:{tracker_type}', self.uuid)
|
|
|
|
# TRACKER LEVEL
|
|
self._set_level(level, tracker_type=tracker_type, user=user_id)
|
|
|
|
# create tracker tags list
|
|
if tags:
|
|
self._set_tags(tags)
|
|
|
|
# create tracker mail notification list
|
|
if mails:
|
|
self._set_mails(mails)
|
|
|
|
# Filters
|
|
if not filters:
|
|
filters = {}
|
|
for obj_type in get_objects_tracked():
|
|
filters[obj_type] = {}
|
|
else:
|
|
self.set_filters(filters)
|
|
for obj_type in filters:
|
|
r_tracker.sadd(f'trackers:objs:{tracker_type}:{obj_type}', to_track)
|
|
r_tracker.sadd(f'trackers:uuid:{tracker_type}:{to_track}', f'{self.uuid}:{obj_type}')
|
|
|
|
self._set_field('last_change', time.time())
|
|
|
|
# toggle refresh module tracker list/set
|
|
trigger_trackers_refresh(tracker_type)
|
|
return self.uuid
|
|
|
|
def edit(self, tracker_type, to_track, level, description=None, filters={}, tags=[], mails=[], webhook=None):
|
|
|
|
# edit tracker
|
|
old_type = self.get_type()
|
|
old_to_track = self.get_tracked()
|
|
old_level = self.get_level()
|
|
user_id = self.get_user()
|
|
|
|
# YARA
|
|
if tracker_type == 'yara_custom' or tracker_type == 'yara_default':
|
|
# create yara rule
|
|
if tracker_type == 'yara_default' and old_type == 'yara':
|
|
if not is_default_yara_rule(old_to_track):
|
|
filepath = get_yara_rule_file_by_tracker_name(old_to_track)
|
|
if filepath:
|
|
os.remove(filepath)
|
|
to_track = save_yara_rule(tracker_type, to_track, tracker_uuid=self.uuid)
|
|
tracker_type = 'yara'
|
|
|
|
# TODO TYPO EDIT
|
|
elif tracker_type == 'typosquatting':
|
|
pass
|
|
|
|
if tracker_type != old_type:
|
|
# LEVEL
|
|
if old_level == 0:
|
|
r_tracker.srem(f'user:tracker:{user_id}:{old_type}', self.uuid)
|
|
elif old_level == 1:
|
|
r_tracker.srem(f'global:tracker:{old_type}', self.uuid)
|
|
self._set_level(level, tracker_type=tracker_type, user=user_id)
|
|
# Delete OLD YARA Rule File
|
|
if old_type == 'yara':
|
|
if not is_default_yara_rule(old_to_track):
|
|
filepath = get_yara_rule_file_by_tracker_name(old_to_track)
|
|
if filepath:
|
|
os.remove(filepath)
|
|
if old_type == 'typosquatting':
|
|
r_tracker.delete(f'tracker:typosquatting:{old_to_track}')
|
|
self._set_field('type', tracker_type)
|
|
|
|
# create all tracker set
|
|
r_tracker.srem(f'all:tracker:{old_type}', old_to_track)
|
|
r_tracker.sadd(f'all:tracker:{tracker_type}', to_track)
|
|
# create tracker - uuid map
|
|
r_tracker.srem(f'all:tracker_uuid:{old_type}:{old_to_track}', self.uuid)
|
|
r_tracker.sadd(f'all:tracker_uuid:{tracker_type}:{to_track}', self.uuid)
|
|
# create all tracker set by type
|
|
r_tracker.srem(f'trackers:all:{old_type}', self.uuid)
|
|
r_tracker.sadd(f'trackers:all:{tracker_type}', self.uuid)
|
|
|
|
# Same Type
|
|
elif level != old_level:
|
|
if level == 0:
|
|
r_tracker.srem('global:tracker', self.uuid)
|
|
elif level == 1:
|
|
r_tracker.srem(f'user:tracker:{user_id}', self.uuid)
|
|
self._set_level(level, tracker_type=tracker_type, user=user_id)
|
|
|
|
# To Track Edited
|
|
if to_track != old_to_track:
|
|
self._set_field('tracked', to_track)
|
|
|
|
self._set_field('description', description)
|
|
self._set_field('webhook', webhook)
|
|
|
|
# Tags
|
|
nb_old_tags = r_tracker.scard(f'tracker:tags:{self.uuid}')
|
|
if nb_old_tags > 0 or tags:
|
|
self._del_tags()
|
|
self._set_tags(tags)
|
|
|
|
# Mails
|
|
nb_old_mails = r_tracker.scard(f'tracker:mail:{self.uuid}')
|
|
if nb_old_mails > 0 or mails:
|
|
self._del_mails()
|
|
self._set_mails(mails)
|
|
|
|
# Filters
|
|
self.del_filters(old_type, old_to_track)
|
|
if not filters:
|
|
filters = {}
|
|
for obj_type in get_objects_tracked():
|
|
filters[obj_type] = {}
|
|
else:
|
|
self.set_filters(filters)
|
|
for obj_type in filters:
|
|
r_tracker.sadd(f'trackers:objs:{tracker_type}:{obj_type}', to_track)
|
|
r_tracker.sadd(f'trackers:uuid:{tracker_type}:{to_track}', f'{self.uuid}:{obj_type}')
|
|
|
|
# Refresh Trackers
|
|
trigger_trackers_refresh(tracker_type)
|
|
if tracker_type != old_type:
|
|
trigger_trackers_refresh(old_type)
|
|
|
|
self._set_field('last_change', time.time())
|
|
return self.uuid
|
|
|
|
def delete(self):
|
|
for obj in self.get_objs():
|
|
self.remove(obj[0], obj[1], obj[2])
|
|
|
|
tracker_type = self.get_type()
|
|
tracked = self.get_tracked()
|
|
r_tracker.srem(f'all:tracker:{tracker_type}', tracked)
|
|
# tracker - uuid map
|
|
r_tracker.srem(f'all:tracker_uuid:{tracker_type}:{tracked}', self.uuid)
|
|
r_tracker.srem('trackers:all', self.uuid)
|
|
r_tracker.srem(f'trackers:all:{tracker_type}', self.uuid)
|
|
|
|
if tracker_type == 'typosquatting':
|
|
r_tracker.delete(f'tracker:typosquatting:{tracked}')
|
|
elif tracker_type == 'yara':
|
|
if not is_default_yara_rule(tracked):
|
|
filepath = get_yara_rule_file_by_tracker_name(tracked)
|
|
if filepath:
|
|
os.remove(filepath)
|
|
|
|
# Filters
|
|
filters = get_objects_tracked()
|
|
for obj_type in filters:
|
|
r_tracker.srem(f'trackers:objs:{tracker_type}:{obj_type}', tracked)
|
|
r_tracker.srem(f'trackers:uuid:{tracker_type}:{tracked}', f'{self.uuid}:{obj_type}')
|
|
|
|
self._del_mails()
|
|
self._del_tags()
|
|
|
|
level = self.get_level()
|
|
|
|
if level == 0: # user only
|
|
user = self.get_user()
|
|
r_tracker.srem(f'user:tracker:{user}', self.uuid)
|
|
r_tracker.srem(f'user:tracker:{user}:{tracker_type}', self.uuid)
|
|
elif level == 1: # global
|
|
r_tracker.srem('global:tracker', self.uuid)
|
|
r_tracker.srem(f'global:tracker:{tracker_type}', self.uuid)
|
|
|
|
# meta
|
|
r_tracker.delete(f'tracker:{self.uuid}')
|
|
trigger_trackers_refresh(tracker_type)
|
|
|
|
|
|
def create_tracker(tracker_type, to_track, user_id, level, description=None, filters={}, tags=[], mails=[], webhook=None, tracker_uuid=None):
|
|
if not tracker_uuid:
|
|
tracker_uuid = str(uuid.uuid4())
|
|
tracker = Tracker(tracker_uuid)
|
|
return tracker.create(tracker_type, to_track, user_id, level, description=description, filters=filters, tags=tags,
|
|
mails=mails, webhook=webhook)
|
|
|
|
def _re_create_tracker(tracker_type, tracker_uuid, to_track, user_id, level, description=None, filters={}, tags=[], mails=[], webhook=None, first_seen=None, last_seen=None):
|
|
create_tracker(tracker_type, to_track, user_id, level, description=description, filters=filters,
|
|
tags=tags, mails=mails, webhook=webhook, tracker_uuid=tracker_uuid)
|
|
|
|
def get_trackers_types():
|
|
return ['word', 'set', 'regex', 'typosquatting', 'yara']
|
|
|
|
def get_trackers():
|
|
return r_tracker.smembers(f'trackers:all')
|
|
|
|
def get_trackers_by_type(tracker_type):
|
|
return r_tracker.smembers(f'trackers:all:{tracker_type}')
|
|
|
|
def _get_tracked_by_obj_type(tracker_type, obj_type):
|
|
return r_tracker.smembers(f'trackers:objs:{tracker_type}:{obj_type}')
|
|
|
|
def get_trackers_by_tracked_obj_type(tracker_type, obj_type, tracked):
|
|
trackers_uuid = set()
|
|
for res in r_tracker.smembers(f'trackers:uuid:{tracker_type}:{tracked}'):
|
|
tracker_uuid, tracker_obj_type = res.split(':', 1)
|
|
if tracker_obj_type == obj_type:
|
|
trackers_uuid.add(tracker_uuid)
|
|
return trackers_uuid
|
|
|
|
def get_trackers_by_tracked(tracker_type, tracked):
|
|
return r_tracker.smembers(f'all:tracker_uuid:{tracker_type}:{tracked}')
|
|
|
|
def get_user_trackers_by_tracked(tracker_type, tracked, user_id):
|
|
user_trackers = get_user_trackers(user_id, tracker_type=tracker_type)
|
|
trackers_uuid = get_trackers_by_tracked(tracker_type, tracked)
|
|
return trackers_uuid.intersection(user_trackers)
|
|
|
|
def get_trackers_tracked_by_type(tracker_type):
|
|
return r_tracker.smembers(f'all:tracker:{tracker_type}')
|
|
|
|
def get_global_trackers(tracker_type=None):
|
|
if tracker_type:
|
|
return r_tracker.smembers(f'global:tracker:{tracker_type}')
|
|
else:
|
|
return r_tracker.smembers('global:tracker')
|
|
|
|
def get_user_trackers(user_id, tracker_type=None):
|
|
if tracker_type:
|
|
return r_tracker.smembers(f'user:tracker:{user_id}:{tracker_type}')
|
|
else:
|
|
return r_tracker.smembers(f'user:tracker:{user_id}')
|
|
|
|
def get_nb_global_trackers(tracker_type=None):
|
|
if tracker_type:
|
|
return r_tracker.scard(f'global:tracker:{tracker_type}')
|
|
else:
|
|
return r_tracker.scard('global:tracker')
|
|
|
|
def get_nb_user_trackers(user_id, tracker_type=None):
|
|
if tracker_type:
|
|
return r_tracker.scard(f'user:tracker:{user_id}:{tracker_type}')
|
|
else:
|
|
return r_tracker.scard(f'user:tracker:{user_id}')
|
|
|
|
def get_user_trackers_meta(user_id, tracker_type=None):
|
|
metas = []
|
|
for tracker_uuid in get_user_trackers(user_id, tracker_type=tracker_type):
|
|
tracker = Tracker(tracker_uuid)
|
|
metas.append(tracker.get_meta(options={'description', 'mails', 'sparkline', 'tags'}))
|
|
return metas
|
|
|
|
def get_global_trackers_meta(tracker_type=None):
|
|
metas = []
|
|
for tracker_uuid in get_global_trackers(tracker_type=tracker_type):
|
|
tracker = Tracker(tracker_uuid)
|
|
metas.append(tracker.get_meta(options={'description', 'mails', 'sparkline', 'tags'}))
|
|
return metas
|
|
|
|
def get_users_trackers_meta():
|
|
trackers = []
|
|
for tracker_uuid in get_trackers():
|
|
tracker = Tracker(tracker_uuid)
|
|
if tracker.is_level_user():
|
|
trackers.append(tracker.get_meta(options={'mails', 'sparkline', 'tags'}))
|
|
return trackers
|
|
|
|
def get_trackers_graph_by_day(l_trackers, num_day=31, date_from=None, date_to=None):
|
|
if date_from and date_to:
|
|
date_range = Date.substract_date(date_from, date_to)
|
|
else:
|
|
date_range = Date.get_date_range(num_day)
|
|
list_tracker_stats = []
|
|
for tracker_uuid in l_trackers:
|
|
dict_tracker_data = []
|
|
tracker = Tracker(tracker_uuid)
|
|
for date_day in date_range:
|
|
nb_seen_this_day = tracker.get_nb_objs_by_date(date_day)
|
|
if nb_seen_this_day is None:
|
|
nb_seen_this_day = 0
|
|
dict_tracker_data.append({"date": date_day, "value": int(nb_seen_this_day)})
|
|
list_tracker_stats.append({"name": tracker.get_tracked(), "Data": dict_tracker_data})
|
|
return list_tracker_stats
|
|
|
|
def get_trackers_dashboard():
|
|
trackers = []
|
|
for raw in r_tracker.lrange('trackers:dashboard', 0, -1):
|
|
tracker_uuid, timestamp, obj_type, subtype, obj_id = raw.split(':', 4)
|
|
tracker = Tracker(tracker_uuid)
|
|
meta = tracker.get_meta(options={'description', 'tags'})
|
|
if not meta.get('type'):
|
|
meta['type'] = 'Tracker DELETED'
|
|
timestamp = datetime.datetime.fromtimestamp(float(timestamp)).strftime('%Y-%m-%d %H:%M:%S')
|
|
meta['timestamp'] = timestamp
|
|
trackers.append(meta)
|
|
return trackers
|
|
|
|
def get_user_dashboard(user_id): # TODO SORT + REMOVE OLDER ROWS (trim)
|
|
trackers = []
|
|
for raw in r_tracker.lrange(f'trackers:user:{user_id}', 0, -1):
|
|
tracker_uuid, timestamp, obj_type, subtype, obj_id = raw.split(':', 4)
|
|
tracker = Tracker(tracker_uuid)
|
|
meta = tracker.get_meta(options={'tags'})
|
|
timestamp = datetime.datetime.fromtimestamp(float(timestamp)).strftime('%Y-%m-%d %H:%M:%S')
|
|
meta['timestamp'] = timestamp
|
|
trackers.append(meta)
|
|
|
|
return trackers
|
|
|
|
def get_trackers_stats(user_id):
|
|
stats = {'all': 0}
|
|
for tracker_type in get_trackers_types():
|
|
nb_global = get_nb_global_trackers(tracker_type=tracker_type)
|
|
nb_user = get_nb_user_trackers(user_id, tracker_type=tracker_type)
|
|
stats[tracker_type] = nb_global + nb_user
|
|
stats['all'] += nb_global + nb_user
|
|
return stats
|
|
|
|
|
|
## Cache ##
|
|
# TODO API: Check Tracker type
|
|
def trigger_trackers_refresh(tracker_type):
|
|
r_cache.set(f'tracker:refresh:{tracker_type}', time.time())
|
|
|
|
def get_tracker_last_updated_by_type(tracker_type):
|
|
epoch_update = r_cache.get(f'tracker:refresh:{tracker_type}')
|
|
if not epoch_update:
|
|
epoch_update = 0
|
|
return float(epoch_update)
|
|
# - Cache - #
|
|
|
|
## Objects ##
|
|
|
|
def is_obj_tracked(obj_type, subtype, obj_id):
|
|
return r_tracker.exists(f'obj:trackers:{obj_type}:{subtype}:{obj_id}')
|
|
|
|
def get_obj_trackers(obj_type, subtype, obj_id):
|
|
return r_tracker.smembers(f'obj:trackers:{obj_type}:{subtype}:{obj_id}')
|
|
|
|
def delete_obj_trackers(obj_type, subtype, obj_id):
|
|
for tracker_uuid in get_obj_trackers(obj_type, subtype, obj_id):
|
|
tracker = Tracker(tracker_uuid)
|
|
tracker.remove(obj_type, subtype, obj_id)
|
|
|
|
######################
|
|
#### TRACKERS ACL ####
|
|
|
|
## LEVEL ##
|
|
def is_tracker_global_level(tracker_uuid):
|
|
return r_tracker.hget(f'tracker:{tracker_uuid}', 'level') == 1
|
|
|
|
def is_tracked_in_global_level(tracked, tracker_type):
|
|
for tracker_uuid in get_trackers_by_tracked(tracker_type, tracked):
|
|
tracker = Tracker(tracker_uuid)
|
|
if tracker.is_level_global():
|
|
return True
|
|
return False
|
|
|
|
def is_tracked_in_user_level(tracked, tracker_type, user_id):
|
|
trackers_uuid = get_user_trackers_by_tracked(tracker_type, tracked, user_id)
|
|
if trackers_uuid:
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
## API ##
|
|
def api_check_tracker_uuid(tracker_uuid):
|
|
if not is_valid_uuid_v4(tracker_uuid):
|
|
return {"status": "error", "reason": "Invalid uuid"}, 400
|
|
if not r_tracker.exists(f'tracker:{tracker_uuid}'):
|
|
return {"status": "error", "reason": "Unknown uuid"}, 404
|
|
return None
|
|
|
|
def api_check_tracker_acl(tracker_uuid, user_id):
|
|
res = api_check_tracker_uuid(tracker_uuid)
|
|
if res:
|
|
return res
|
|
tracker = Tracker(tracker_uuid)
|
|
if tracker.is_level_user():
|
|
if tracker.get_user() != user_id or not User(user_id).is_in_role('admin'):
|
|
return {"status": "error", "reason": "Access Denied"}, 403
|
|
return None
|
|
|
|
def api_is_allowed_to_edit_tracker(tracker_uuid, user_id):
|
|
if not is_valid_uuid_v4(tracker_uuid):
|
|
return {"status": "error", "reason": "Invalid uuid"}, 400
|
|
tracker_creator = r_tracker.hget('tracker:{}'.format(tracker_uuid), 'user_id')
|
|
if not tracker_creator:
|
|
return {"status": "error", "reason": "Unknown uuid"}, 404
|
|
user = User(user_id)
|
|
if not user.is_in_role('admin') and user_id != tracker_creator:
|
|
return {"status": "error", "reason": "Access Denied"}, 403
|
|
return {"uuid": tracker_uuid}, 200
|
|
|
|
|
|
def api_is_allowed_to_access_tracker(tracker_uuid, user_id):
|
|
if not is_valid_uuid_v4(tracker_uuid):
|
|
return {"status": "error", "reason": "Invalid uuid"}, 400
|
|
tracker_creator = r_tracker.hget('tracker:{}'.format(tracker_uuid), 'user_id')
|
|
if not tracker_creator:
|
|
return {"status": "error", "reason": "Unknown uuid"}, 404
|
|
user = User(user_id)
|
|
if not is_tracker_global_level(tracker_uuid):
|
|
if not user.is_in_role('admin') and user_id != tracker_creator:
|
|
return {"status": "error", "reason": "Access Denied"}, 403
|
|
return {"uuid": tracker_uuid}, 200
|
|
|
|
##-- ACL --##
|
|
|
|
#### FIX DB #### TODO ###################################################################
|
|
def fix_tracker_stats_per_day(tracker_uuid):
|
|
tracker = Tracker(tracker_uuid)
|
|
date_from = tracker.get_date()
|
|
date_to = Date.get_today_date_str()
|
|
# delete stats
|
|
r_tracker.delete(f'tracker:stat:{tracker_uuid}')
|
|
r_tracker.hdel(f'tracker:{tracker_uuid}', 'first_seen')
|
|
r_tracker.hdel(f'tracker:{tracker_uuid}', 'last_seen')
|
|
# create new stats
|
|
for date_day in Date.substract_date(date_from, date_to):
|
|
date_day = int(date_day)
|
|
|
|
nb_items = r_tracker.scard(f'tracker:item:{tracker_uuid}:{date_day}')
|
|
if nb_items:
|
|
r_tracker.zincrby(f'tracker:stat:{tracker_uuid}', nb_items, int(date_day))
|
|
|
|
# update first_seen/last_seen
|
|
tracker.update_daterange(date_day)
|
|
|
|
def fix_tracker_item_link(tracker_uuid):
|
|
tracker = Tracker(tracker_uuid)
|
|
date_from = tracker.get_first_seen()
|
|
date_to = tracker.get_last_seen()
|
|
|
|
if date_from and date_to:
|
|
for date_day in Date.substract_date(date_from, date_to):
|
|
l_items = r_tracker.smembers(f'tracker:item:{tracker_uuid}:{date_day}')
|
|
for item_id in l_items:
|
|
r_tracker.sadd(f'obj:trackers:item:{item_id}', tracker_uuid)
|
|
|
|
def fix_all_tracker_uuid_list():
|
|
r_tracker.delete(f'trackers:all')
|
|
for tracker_type in get_trackers_types():
|
|
r_tracker.delete(f'trackers:all:{tracker_type}')
|
|
for tracked in get_trackers_tracked_by_type(tracker_type):
|
|
l_tracker_uuid = get_trackers_by_tracked(tracker_type, tracked)
|
|
for tracker_uuid in l_tracker_uuid:
|
|
r_tracker.sadd(f'trackers:all', tracker_uuid)
|
|
r_tracker.sadd(f'trackers:all:{tracker_type}', tracker_uuid)
|
|
|
|
##-- FIX DB --##
|
|
|
|
#### CREATE TRACKER ####
|
|
def api_validate_tracker_to_add(to_track, tracker_type, nb_words=1):
|
|
if tracker_type == 'regex':
|
|
if not is_valid_regex(to_track):
|
|
return {"status": "error", "reason": "Invalid regex"}, 400
|
|
elif tracker_type == 'word' or tracker_type == 'set':
|
|
# force lowercase
|
|
to_track = to_track.lower()
|
|
word_set = set(to_track)
|
|
set_inter = word_set.intersection(get_special_characters())
|
|
if set_inter:
|
|
return {"status": "error",
|
|
"reason": f'special character(s) not allowed: {set_inter}',
|
|
"message": "Please use a python regex or remove all special characters"}, 400
|
|
words = to_track.split()
|
|
# not a word
|
|
if tracker_type == 'word' and len(words) > 1:
|
|
tracker_type = 'set'
|
|
|
|
# output format: tracker1,tracker2,tracker3;2
|
|
if tracker_type == 'set':
|
|
try:
|
|
nb_words = int(nb_words)
|
|
except TypeError:
|
|
nb_words = 1
|
|
if nb_words == 0:
|
|
nb_words = 1
|
|
|
|
words_set = set(words)
|
|
words_set = sorted(words_set)
|
|
if nb_words > len(words_set):
|
|
nb_words = len(words_set)
|
|
|
|
to_track = ",".join(words_set)
|
|
to_track = f"{to_track};{nb_words}"
|
|
elif tracker_type == 'typosquatting':
|
|
to_track = to_track.lower()
|
|
# Take only the first term
|
|
domain = to_track.split(" ")
|
|
if len(domain) > 1:
|
|
return {"status": "error", "reason": "Only one domain is accepted at a time"}, 400
|
|
if "." not in to_track:
|
|
return {"status": "error", "reason": "Invalid domain name"}, 400
|
|
|
|
elif tracker_type == 'yara_custom':
|
|
if not is_valid_yara_rule(to_track):
|
|
return {"status": "error", "reason": "Invalid custom Yara Rule"}, 400
|
|
elif tracker_type == 'yara_default':
|
|
if not is_valid_default_yara_rule(to_track):
|
|
return {"status": "error", "reason": "The Yara Rule doesn't exist"}, 400
|
|
else:
|
|
return {"status": "error", "reason": "Incorrect type"}, 400
|
|
return {"status": "success", "tracked": to_track, "type": tracker_type}, 200
|
|
|
|
def api_add_tracker(dict_input, user_id):
|
|
to_track = dict_input.get('tracked', None)
|
|
if not to_track:
|
|
return {"status": "error", "reason": "Tracker not provided"}, 400
|
|
tracker_type = dict_input.get('type', None)
|
|
if not tracker_type:
|
|
return {"status": "error", "reason": "Tracker type not provided"}, 400
|
|
nb_words = dict_input.get('nb_words', 1)
|
|
description = dict_input.get('description', '')
|
|
description = escape(description)
|
|
webhook = dict_input.get('webhook', '')
|
|
webhook = escape(webhook)
|
|
res = api_validate_tracker_to_add(to_track, tracker_type, nb_words=nb_words)
|
|
if res[1] != 200:
|
|
return res
|
|
to_track = res[0]['tracked']
|
|
tracker_type = res[0]['type']
|
|
|
|
tags = dict_input.get('tags', [])
|
|
mails = dict_input.get('mails', [])
|
|
res = verify_mail_list(mails)
|
|
if res:
|
|
return res
|
|
|
|
# Filters # TODO MOVE ME
|
|
filters = dict_input.get('filters', {})
|
|
if filters:
|
|
if filters.keys() == set(get_objects_tracked()) and set(filters['pgp'].get('subtypes', [])) == {'mail', 'name'}:
|
|
filters = {}
|
|
for obj_type in filters:
|
|
if obj_type not in get_objects_tracked():
|
|
return {"status": "error", "reason": "Invalid Tracker Object type"}, 400
|
|
|
|
if obj_type == 'pgp':
|
|
if set(filters['pgp'].get('subtypes', [])) == {'mail', 'name'}:
|
|
filters['pgp'].pop('subtypes')
|
|
|
|
for filter_name in filters[obj_type]:
|
|
if filter_name not in {'mimetypes', 'sources', 'subtypes'}:
|
|
return {"status": "error", "reason": "Invalid Filter"}, 400
|
|
elif filter_name == 'mimetypes': # TODO
|
|
pass
|
|
elif filter_name == 'sources':
|
|
if obj_type == 'item':
|
|
res = item_basic.verify_sources_list(filters['item']['sources'])
|
|
if res:
|
|
return res
|
|
else:
|
|
return {"status": "error", "reason": "Invalid Filter sources"}, 400
|
|
elif filter_name == 'subtypes':
|
|
obj_subtypes = set(get_object_all_subtypes(obj_type))
|
|
for subtype in filters[obj_type]['subtypes']:
|
|
if subtype not in obj_subtypes:
|
|
return {"status": "error", "reason": "Invalid Tracker Object subtype"}, 400
|
|
|
|
level = dict_input.get('level', 1)
|
|
try:
|
|
level = int(level)
|
|
except TypeError:
|
|
level = 1
|
|
if level not in range(0, 1):
|
|
level = 1
|
|
|
|
tracker_uuid = create_tracker(tracker_type, to_track, user_id, level, description=description, filters=filters,
|
|
tags=tags, mails=mails, webhook=webhook)
|
|
|
|
return {'tracked': to_track, 'type': tracker_type, 'uuid': tracker_uuid}, 200
|
|
|
|
def api_edit_tracker(dict_input, user_id):
|
|
tracker_uuid = dict_input.get('uuid')
|
|
res = api_check_tracker_acl(tracker_uuid, user_id)
|
|
if res:
|
|
return res
|
|
|
|
tracker = Tracker(tracker_uuid)
|
|
|
|
to_track = dict_input.get('tracked', None)
|
|
if not to_track:
|
|
return {"status": "error", "reason": "Tracker not provided"}, 400
|
|
tracker_type = dict_input.get('type', None)
|
|
if not tracker_type:
|
|
return {"status": "error", "reason": "Tracker type not provided"}, 400
|
|
nb_words = dict_input.get('nb_words', 1)
|
|
description = dict_input.get('description', '')
|
|
description = escape(description)
|
|
webhook = dict_input.get('webhook', '')
|
|
webhook = escape(webhook)
|
|
res = api_validate_tracker_to_add(to_track, tracker_type, nb_words=nb_words)
|
|
if res[1] != 200:
|
|
return res
|
|
to_track = res[0]['tracked']
|
|
tracker_type = res[0]['type']
|
|
|
|
tags = dict_input.get('tags', [])
|
|
mails = dict_input.get('mails', [])
|
|
res = verify_mail_list(mails)
|
|
if res:
|
|
return res
|
|
|
|
# Filters # TODO MOVE ME
|
|
filters = dict_input.get('filters', {})
|
|
if filters:
|
|
if filters.keys() == set(get_objects_tracked()) and set(filters['pgp'].get('subtypes', [])) == {'mail', 'name'}:
|
|
if not filters['decoded'] and not filters['item']:
|
|
filters = {}
|
|
for obj_type in filters:
|
|
if obj_type not in get_objects_tracked():
|
|
return {"status": "error", "reason": "Invalid Tracker Object type"}, 400
|
|
|
|
if obj_type == 'pgp':
|
|
if set(filters['pgp'].get('subtypes', [])) == {'mail', 'name'}:
|
|
filters['pgp'].pop('subtypes')
|
|
|
|
for filter_name in filters[obj_type]:
|
|
if filter_name not in {'mimetypes', 'sources', 'subtypes'}:
|
|
return {"status": "error", "reason": "Invalid Filter"}, 400
|
|
elif filter_name == 'mimetypes': # TODO
|
|
pass
|
|
elif filter_name == 'sources':
|
|
if obj_type == 'item':
|
|
res = item_basic.verify_sources_list(filters['item']['sources'])
|
|
if res:
|
|
return res
|
|
else:
|
|
return {"status": "error", "reason": "Invalid Filter sources"}, 400
|
|
elif filter_name == 'subtypes':
|
|
obj_subtypes = set(get_object_all_subtypes(obj_type))
|
|
for subtype in filters[obj_type]['subtypes']:
|
|
if subtype not in obj_subtypes:
|
|
return {"status": "error", "reason": "Invalid Tracker Object subtype"}, 400
|
|
|
|
level = dict_input.get('level', 1)
|
|
try:
|
|
level = int(level)
|
|
except TypeError:
|
|
level = 1
|
|
if level not in range(0, 1):
|
|
level = 1
|
|
|
|
tracker.edit(tracker_type, to_track, level, description=description, filters=filters,
|
|
tags=tags, mails=mails, webhook=webhook)
|
|
return {'tracked': to_track, 'type': tracker_type, 'uuid': tracker_uuid}, 200
|
|
|
|
|
|
def api_delete_tracker(data, user_id):
|
|
tracker_uuid = data.get('uuid')
|
|
res = api_check_tracker_acl(tracker_uuid, user_id)
|
|
if res:
|
|
return res
|
|
|
|
tracker = Tracker(tracker_uuid)
|
|
return tracker.delete(), 200
|
|
|
|
def api_tracker_add_object(data, user_id):
|
|
tracker_uuid = data.get('uuid')
|
|
res = api_check_tracker_acl(tracker_uuid, user_id)
|
|
if res:
|
|
return res
|
|
tracker = Tracker(tracker_uuid)
|
|
object_gid = data.get('gid')
|
|
date = data.get('date')
|
|
if date:
|
|
if not Date.validate_str_date(date):
|
|
date = None
|
|
try:
|
|
obj_type, subtype, obj_id = object_gid.split(':', 2)
|
|
except (AttributeError, IndexError):
|
|
return {"status": "error", "reason": "Invalid Object"}, 400
|
|
return tracker.add(obj_type, subtype, obj_id, date=date), 200
|
|
|
|
def api_tracker_remove_object(data, user_id):
|
|
tracker_uuid = data.get('uuid')
|
|
res = api_check_tracker_acl(tracker_uuid, user_id)
|
|
if res:
|
|
return res
|
|
|
|
tracker = Tracker(tracker_uuid)
|
|
object_gid = data.get('gid')
|
|
try:
|
|
obj_type, subtype, obj_id = object_gid.split(':', 2)
|
|
except (AttributeError, IndexError):
|
|
return {"status": "error", "reason": "Invalid Object"}, 400
|
|
return tracker.remove(obj_type, subtype, obj_id), 200
|
|
|
|
## -- CREATE TRACKER -- ##
|
|
|
|
####################
|
|
#### WORD - SET ####
|
|
|
|
def get_tracked_words():
|
|
to_track = {}
|
|
for obj_type in get_objects_tracked():
|
|
to_track[obj_type] = _get_tracked_by_obj_type('word', obj_type)
|
|
return to_track
|
|
|
|
def get_tracked_sets():
|
|
to_track = {}
|
|
for obj_type in get_objects_tracked():
|
|
to_track[obj_type] = []
|
|
for tracked in _get_tracked_by_obj_type('set', obj_type):
|
|
res = tracked.split(';')
|
|
nb_words = int(res[1])
|
|
words_set = res[0].split(',')
|
|
to_track[obj_type].append({'words': words_set, 'nb': nb_words, 'tracked': tracked})
|
|
return to_track
|
|
|
|
def get_text_word_frequency(content, filtering=True):
|
|
content = content.lower()
|
|
words_dict = defaultdict(int)
|
|
|
|
if filtering:
|
|
if TOKENIZER is None:
|
|
init_tokenizer()
|
|
blob = TextBlob(content, tokenizer=TOKENIZER)
|
|
else:
|
|
blob = TextBlob(content)
|
|
for word in blob.tokens:
|
|
words_dict[word] += 1
|
|
return words_dict
|
|
|
|
###############
|
|
#### REGEX ####
|
|
|
|
def get_tracked_regexs():
|
|
to_track = {}
|
|
for obj_type in get_objects_tracked():
|
|
to_track[obj_type] = []
|
|
for tracked in _get_tracked_by_obj_type('regex', obj_type):
|
|
to_track[obj_type].append({'regex': re.compile(tracked), 'tracked': tracked})
|
|
return to_track
|
|
|
|
########################
|
|
#### TYPO SQUATTING ####
|
|
|
|
def get_tracked_typosquatting_domains(tracked):
|
|
return r_tracker.smembers(f'tracker:typosquatting:{tracked}')
|
|
|
|
def get_tracked_typosquatting():
|
|
to_track = {}
|
|
for obj_type in get_objects_tracked():
|
|
to_track[obj_type] = []
|
|
for tracked in _get_tracked_by_obj_type('typosquatting', obj_type):
|
|
to_track[obj_type].append({'domains': get_tracked_typosquatting_domains(tracked), 'tracked': tracked})
|
|
return to_track
|
|
|
|
##############
|
|
#### YARA ####
|
|
def get_yara_rules_dir():
|
|
return os.path.join(os.environ['AIL_BIN'], 'trackers', 'yara')
|
|
|
|
def get_yara_rules_default_dir():
|
|
return os.path.join(os.environ['AIL_BIN'], 'trackers', 'yara', 'ail-yara-rules', 'rules')
|
|
|
|
# # TODO: cache + update
|
|
def get_all_default_yara_rules_types():
|
|
yara_dir = get_yara_rules_default_dir()
|
|
all_yara_types = next(os.walk(yara_dir))[1]
|
|
# save in cache ?
|
|
return all_yara_types
|
|
|
|
# # TODO: cache + update
|
|
def get_all_default_yara_files():
|
|
yara_dir = get_yara_rules_default_dir()
|
|
all_default_yara_files = {}
|
|
for rules_type in get_all_default_yara_rules_types():
|
|
all_default_yara_files[rules_type] = os.listdir(os.path.join(yara_dir, rules_type))
|
|
return all_default_yara_files
|
|
|
|
def get_all_default_yara_rules_by_type(yara_types):
|
|
all_default_yara_files = get_all_default_yara_files()
|
|
if yara_types in all_default_yara_files:
|
|
return all_default_yara_files[yara_types]
|
|
else:
|
|
return []
|
|
|
|
def get_all_tracked_yara_files(filter_disabled=False):
|
|
yara_files = r_tracker.smembers('all:tracker:yara')
|
|
if not yara_files:
|
|
yara_files = []
|
|
if filter_disabled:
|
|
pass
|
|
return yara_files
|
|
|
|
def get_tracked_yara_rules():
|
|
to_track = {}
|
|
for obj_type in get_objects_tracked():
|
|
rules = {}
|
|
for tracked in _get_tracked_by_obj_type('yara', obj_type):
|
|
rule = os.path.join(get_yara_rules_dir(), tracked)
|
|
if not os.path.exists(rule):
|
|
logger.critical(f"Yara rule don't exists {tracked} : {obj_type}")
|
|
else:
|
|
rules[tracked] = rule
|
|
to_track[obj_type] = yara.compile(filepaths=rules)
|
|
print(to_track)
|
|
return to_track
|
|
|
|
def reload_yara_rules():
|
|
yara_files = get_all_tracked_yara_files()
|
|
# {uuid: filename}
|
|
rule_dict = {}
|
|
for yar_path in yara_files:
|
|
for tracker_uuid in get_trackers_by_tracked('yara', yar_path):
|
|
rule_dict[tracker_uuid] = os.path.join(get_yara_rules_dir(), yar_path)
|
|
for tracker_uuid in rule_dict:
|
|
if not os.path.isfile(rule_dict[tracker_uuid]):
|
|
# TODO IGNORE + LOGS
|
|
raise Exception(f"Error: {rule_dict[tracker_uuid]} doesn't exists")
|
|
rules = yara.compile(filepaths=rule_dict)
|
|
return rules
|
|
|
|
def is_valid_yara_rule(yara_rule):
|
|
try:
|
|
yara.compile(source=yara_rule)
|
|
return True
|
|
except:
|
|
return False
|
|
|
|
def is_default_yara_rule(tracked_yara_name):
|
|
yara_dir = get_yara_rules_dir()
|
|
filename = os.path.join(yara_dir, tracked_yara_name)
|
|
filename = os.path.realpath(filename)
|
|
try:
|
|
if tracked_yara_name.split('/')[0] == 'custom-rules':
|
|
return False
|
|
except:
|
|
return False
|
|
if not os.path.commonprefix([filename, yara_dir]) == yara_dir:
|
|
return False
|
|
else:
|
|
if os.path.isfile(filename):
|
|
return True
|
|
return False
|
|
|
|
def is_valid_default_yara_rule(yara_rule, verbose=True):
|
|
yara_dir = get_yara_rules_default_dir()
|
|
filename = os.path.join(yara_dir, yara_rule)
|
|
filename = os.path.realpath(filename)
|
|
# incorrect filename
|
|
if not os.path.commonprefix([filename, yara_dir]) == yara_dir:
|
|
if verbose:
|
|
print('error: file transversal')
|
|
print(yara_dir)
|
|
print(filename)
|
|
return False
|
|
else:
|
|
if os.path.isfile(filename):
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
def save_yara_rule(yara_rule_type, yara_rule, tracker_uuid=None):
|
|
if yara_rule_type == 'yara_custom':
|
|
if not tracker_uuid:
|
|
tracker_uuid = str(uuid.uuid4())
|
|
filename = os.path.join('custom-rules', tracker_uuid + '.yar')
|
|
with open(os.path.join(get_yara_rules_dir(), filename), 'w') as f:
|
|
f.write(str(yara_rule))
|
|
elif yara_rule_type == 'yara_default':
|
|
filename = os.path.join('ail-yara-rules', 'rules', yara_rule)
|
|
return filename
|
|
|
|
def get_yara_rule_file_by_tracker_name(tracked_yara_name):
|
|
yara_dir = get_yara_rules_dir()
|
|
filename = os.path.join(yara_dir, tracked_yara_name)
|
|
filename = os.path.realpath(filename)
|
|
if not os.path.commonprefix([filename, yara_dir]) == yara_dir:
|
|
print('error: file transversal')
|
|
print(yara_dir)
|
|
print(filename)
|
|
return None
|
|
return filename
|
|
|
|
def get_yara_rule_content(yara_rule):
|
|
yara_dir = get_yara_rules_dir()
|
|
filename = os.path.join(yara_dir, yara_rule)
|
|
filename = os.path.realpath(filename)
|
|
|
|
# incorrect filename
|
|
if not os.path.commonprefix([filename, yara_dir]) == yara_dir:
|
|
return '' # # TODO: throw exception
|
|
|
|
with open(filename, 'r') as f:
|
|
rule_content = f.read()
|
|
return rule_content
|
|
|
|
def api_get_default_rule_content(default_yara_rule):
|
|
yara_dir = get_yara_rules_default_dir()
|
|
filename = os.path.join(yara_dir, default_yara_rule)
|
|
filename = os.path.realpath(filename)
|
|
if not os.path.commonprefix([filename, yara_dir]) == yara_dir:
|
|
return {'status': 'error', 'reason': 'file traversal detected'}, 400
|
|
|
|
if not os.path.isfile(filename):
|
|
return {'status': 'error', 'reason': 'yara rule not found'}, 400
|
|
|
|
with open(filename, 'r') as f:
|
|
rule_content = f.read()
|
|
return {'rule_name': default_yara_rule, 'content': rule_content}, 200
|
|
|
|
|
|
def get_yara_rule_content_restapi(request_dict):
|
|
rule_name = request_dict.get('rule_name', None)
|
|
if not request_dict:
|
|
return {'status': 'error', 'reason': 'Malformed JSON'}, 400
|
|
if not rule_name:
|
|
return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400
|
|
yara_dir = get_yara_rules_dir()
|
|
filename = os.path.join(yara_dir, rule_name)
|
|
filename = os.path.realpath(filename)
|
|
if not os.path.commonprefix([filename, yara_dir]) == yara_dir:
|
|
return {'status': 'error', 'reason': 'File Path Traversal'}, 400
|
|
if not os.path.isfile(filename):
|
|
return {'status': 'error', 'reason': 'yara rule not found'}, 400
|
|
with open(filename, 'r') as f:
|
|
rule_content = f.read()
|
|
rule_content = base64.b64encode((rule_content.encode('utf-8'))).decode('UTF-8')
|
|
return {'status': 'success', 'content': rule_content}, 200
|
|
|
|
## -- YARA -- ##
|
|
|
|
######################
|
|
#### RETRO - HUNT ####
|
|
|
|
# state: pending/running/completed/paused
|
|
|
|
class RetroHunt:
|
|
|
|
def __init__(self, task_uuid):
|
|
self.uuid = task_uuid
|
|
|
|
def exists(self):
|
|
return r_tracker.exists(f'retro_hunt:{self.uuid}')
|
|
|
|
def _get_field(self, field):
|
|
return r_tracker.hget(f'retro_hunt:{self.uuid}', field)
|
|
|
|
def _set_field(self, field, value):
|
|
return r_tracker.hset(f'retro_hunt:{self.uuid}', field, value)
|
|
|
|
def get_creator(self):
|
|
return self._get_field('creator')
|
|
|
|
def get_date(self):
|
|
return self._get_field('date')
|
|
|
|
def get_last_analyzed(self):
|
|
return self._get_field('last')
|
|
|
|
def set_last_analyzed(self, obj_type, subtype, obj_id):
|
|
return self._set_field('last', f'{obj_type}:{subtype}:{obj_id}')
|
|
|
|
def get_last_analyzed_cache(self):
|
|
r_cache.hget(f'retro_hunt:task:{self.uuid}', 'obj')
|
|
|
|
def set_last_analyzed_cache(self, obj_type, subtype, obj_id):
|
|
r_cache.hset(f'retro_hunt:task:{self.uuid}', 'obj', f'{obj_type}:{subtype}:{obj_id}')
|
|
|
|
def get_name(self):
|
|
return self._get_field('name')
|
|
|
|
def get_description(self):
|
|
return self._get_field('description')
|
|
|
|
def get_timeout(self):
|
|
res = self._get_field('timeout')
|
|
if res:
|
|
return int(res)
|
|
else:
|
|
return 30 # # TODO: FIXME use instance limit
|
|
|
|
def get_filters(self):
|
|
filters = self._get_field('filters')
|
|
if not filters:
|
|
return {}
|
|
else:
|
|
return json.loads(filters)
|
|
|
|
def set_filters(self, filters):
|
|
if filters:
|
|
self._set_field('filters', json.dumps(filters))
|
|
|
|
def get_tags(self):
|
|
return r_tracker.smembers(f'retro_hunt:tags:{self.uuid}')
|
|
|
|
def get_mails(self):
|
|
return r_tracker.smembers(f'retro_hunt:mails:{self.uuid}')
|
|
|
|
def get_state(self):
|
|
return self._get_field('state')
|
|
|
|
def _set_state(self, new_state):
|
|
curr_state = self.get_state()
|
|
if curr_state:
|
|
r_tracker.srem(f'retro_hunt:task:{curr_state}', self.uuid)
|
|
r_tracker.sadd(f'retro_hunts:{new_state}', self.uuid)
|
|
self._set_field('state', new_state)
|
|
|
|
def get_rule(self, r_compile=False):
|
|
rule = self._get_field('rule')
|
|
if r_compile:
|
|
rule = os.path.join(get_yara_rules_dir(), rule)
|
|
rule_dict = {self.uuid: os.path.join(get_yara_rules_dir(), rule)}
|
|
rule = yara.compile(filepaths=rule_dict)
|
|
return rule
|
|
|
|
# add timeout ?
|
|
def get_meta(self, options=set()):
|
|
meta = {'uuid': self.uuid,
|
|
'name': self.get_name(),
|
|
'state': self.get_state(),
|
|
'rule': self.get_rule(),
|
|
}
|
|
if 'creator' in options:
|
|
meta['creator'] = self.get_creator()
|
|
if 'date' in options:
|
|
meta['date'] = self.get_date()
|
|
if 'description' in options:
|
|
meta['description'] = self.get_description()
|
|
if 'mails' in options:
|
|
meta['mails'] = self.get_mails()
|
|
if 'nb_match' in options:
|
|
meta['nb_match'] = self.get_nb_match()
|
|
if 'nb_objs' in options:
|
|
meta['nb_objs'] = self.get_nb_objs()
|
|
if 'progress' in options:
|
|
meta['progress'] = self.get_progress()
|
|
if 'filters' in options:
|
|
meta['filters'] = self.get_filters()
|
|
if 'tags' in options:
|
|
meta['tags'] = self.get_tags()
|
|
return meta
|
|
|
|
def is_paused(self):
|
|
return r_tracker.sismember('retro_hunts:paused', self.uuid)
|
|
|
|
def to_pause(self):
|
|
to_pause = r_cache.hget(f'retro_hunt:{self.uuid}', 'pause')
|
|
if to_pause:
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
def pause(self):
|
|
self._set_state('paused')
|
|
r_cache.hset(f'retro_hunt:{self.uuid}', 'pause', time.time())
|
|
self.clear_cache()
|
|
|
|
def resume(self):
|
|
r_cache.hdel(f'retro_hunt:{self.uuid}', 'pause')
|
|
self._set_state('pending')
|
|
|
|
def is_running(self):
|
|
return r_tracker.sismember('retro_hunts:running', self.uuid)
|
|
|
|
def run(self): # TODO ADD MORE CHECK
|
|
self._set_state('running')
|
|
|
|
def complete(self):
|
|
self._set_state('completed')
|
|
self.clear_cache()
|
|
r_tracker.hdel(f'retro_hunt:{self.uuid}', 'last')
|
|
|
|
def get_progress(self):
|
|
if self.get_state() == 'completed':
|
|
progress = 100
|
|
else:
|
|
progress = r_cache.hget(f'retro_hunt:{self.uuid}', 'progress')
|
|
if not progress:
|
|
progress = self._get_field('progress')
|
|
return progress
|
|
|
|
def set_progress(self, progress):
|
|
res = r_cache.hset(f'retro_hunt:{self.uuid}', 'progress', progress)
|
|
if res:
|
|
self._set_field('progress', progress)
|
|
|
|
def get_nb_match(self):
|
|
return self._get_field('nb_match')
|
|
|
|
def _incr_nb_match(self):
|
|
r_tracker.hincrby(f'retro_hunt:{self.uuid}', 'nb_match', 1)
|
|
|
|
def _decr_nb_match(self):
|
|
r_tracker.hincrby(f'retro_hunt:{self.uuid}', 'nb_match', -1)
|
|
|
|
def _set_nb_match(self, nb_match):
|
|
self._set_field('nb_match', nb_match)
|
|
|
|
def clear_cache(self):
|
|
r_cache.delete(f'retro_hunt:{self.uuid}')
|
|
|
|
def get_nb_objs_by_type(self, obj_type):
|
|
return r_tracker.scard(f'retro_hunt:objs:{self.uuid}:{obj_type}')
|
|
|
|
def get_objs_by_type(self, obj_type):
|
|
return r_tracker.smembers(f'retro_hunt:objs:{self.uuid}:{obj_type}')
|
|
|
|
def get_nb_objs(self):
|
|
objs = {}
|
|
for obj_type in get_objects_retro_hunted():
|
|
nb = self.get_nb_objs_by_type(obj_type)
|
|
if nb:
|
|
objs[obj_type] = nb
|
|
return objs
|
|
|
|
def get_objs(self):
|
|
objs = []
|
|
for obj_type in get_objects_retro_hunted():
|
|
for obj in self.get_objs_by_type(obj_type):
|
|
subtype, obj_id = obj.split(':', 1)
|
|
objs.append((obj_type, subtype, obj_id))
|
|
return objs
|
|
|
|
def add(self, obj_type, subtype, obj_id):
|
|
# match by object type:
|
|
r_tracker.sadd(f'retro_hunt:objs:{self.uuid}:{obj_type}', f'{subtype}:{obj_id}')
|
|
# MAP object -> retro hunt
|
|
r_tracker.sadd(f'obj:retro_hunts:{obj_type}:{subtype}:{obj_id}', self.uuid)
|
|
self._incr_nb_match()
|
|
|
|
def remove(self, obj_type, subtype, obj_id):
|
|
# match by object type:
|
|
r_tracker.srem(f'retro_hunt:objs:{self.uuid}:{obj_type}', f'{subtype}:{obj_id}')
|
|
# MAP object -> retro hunt
|
|
r_tracker.srem(f'obj:retro_hunts:{obj_type}:{subtype}:{obj_id}', self.uuid)
|
|
self._decr_nb_match()
|
|
|
|
def create(self, name, rule, creator, description=None, filters=[], mails=[], tags=[], timeout=30, state='pending'):
|
|
if self.exists():
|
|
raise Exception('Error: Retro Hunt Task already exists')
|
|
|
|
self._set_field('name', escape(name))
|
|
|
|
self._set_field('rule', rule)
|
|
|
|
self._set_field('date', datetime.date.today().strftime("%Y%m%d"))
|
|
self._set_field('name', escape(name))
|
|
self._set_field('creator', creator)
|
|
if description:
|
|
self._set_field('description', description)
|
|
if timeout:
|
|
self._set_field('timeout', int(timeout))
|
|
for tag in tags:
|
|
# tag = escape(tag)
|
|
r_tracker.sadd(f'retro_hunt:tags:{self.uuid}', tag)
|
|
Tag.create_custom_tag(tag)
|
|
for mail in mails:
|
|
r_tracker.sadd(f'retro_hunt:mails:{self.uuid}', escape(mail))
|
|
|
|
if filters:
|
|
self.set_filters(filters)
|
|
|
|
r_tracker.sadd('retro_hunts:all', self.uuid)
|
|
|
|
# add to pending tasks
|
|
if state not in ('pending', 'completed', 'paused'):
|
|
state = 'pending'
|
|
self._set_state(state)
|
|
|
|
def delete(self):
|
|
if self.is_running() and self.get_state() not in ['completed', 'paused']:
|
|
return None
|
|
|
|
# Delete custom rule
|
|
rule = self.get_rule()
|
|
if not is_default_yara_rule(rule):
|
|
filepath = get_yara_rule_file_by_tracker_name(rule)
|
|
if filepath:
|
|
os.remove(filepath)
|
|
|
|
r_tracker.srem('retro_hunts:pending', self.uuid)
|
|
r_tracker.delete(f'retro_hunts:{self.uuid}')
|
|
r_tracker.delete(f'retro_hunt:tags:{self.uuid}')
|
|
r_tracker.delete(f'retro_hunt:mails:{self.uuid}')
|
|
|
|
for obj in self.get_objs():
|
|
self.remove(obj[0], obj[1], obj[2])
|
|
|
|
r_tracker.srem('retro_hunts:all', self.uuid)
|
|
r_tracker.srem('retro_hunts:pending', self.uuid)
|
|
r_tracker.srem('retro_hunts:paused', self.uuid)
|
|
r_tracker.srem('retro_hunts:completed', self.uuid)
|
|
|
|
self.clear_cache()
|
|
return self.uuid
|
|
|
|
def create_retro_hunt(name, rule_type, rule, creator, description=None, filters=[], mails=[], tags=[], timeout=30, state='pending', task_uuid=None):
|
|
if not task_uuid:
|
|
task_uuid = str(uuid.uuid4())
|
|
retro_hunt = RetroHunt(task_uuid)
|
|
# rule_type: yara_default - yara custom
|
|
rule = save_yara_rule(rule_type, rule, tracker_uuid=retro_hunt.uuid)
|
|
retro_hunt.create(name, rule, creator, description=description, mails=mails, tags=tags,
|
|
timeout=timeout, filters=filters, state=state)
|
|
return retro_hunt.uuid
|
|
|
|
# TODO
|
|
# def _re_create_retro_hunt_task(name, rule, date, date_from, date_to, creator, sources, tags, mails, timeout, description, task_uuid, state='pending', nb_match=0, last_id=None):
|
|
# retro_hunt = RetroHunt(task_uuid)
|
|
# retro_hunt.create(name, rule, date_from, date_to, creator, description=description, mails=mails, tags=tags,
|
|
# timeout=timeout, sources=sources, state=state)
|
|
# if last_id:
|
|
# set_retro_hunt_last_analyzed(task_uuid, last_id)
|
|
# retro_hunt._set_nb_match(nb_match)
|
|
# retro_hunt._set_field('date', date)
|
|
|
|
## ? ? ?
|
|
# set tags
|
|
# set mails
|
|
# limit mail
|
|
|
|
# SET Retro Hunts
|
|
|
|
def get_all_retro_hunt_tasks():
|
|
return r_tracker.smembers('retro_hunts:all')
|
|
|
|
def get_retro_hunt_pending_tasks():
|
|
return r_tracker.smembers('retro_hunts:pending')
|
|
|
|
def get_retro_hunt_running_tasks():
|
|
return r_tracker.smembers('retro_hunts:running')
|
|
|
|
def get_retro_hunt_paused_tasks():
|
|
return r_tracker.smembers('retro_hunts:paused')
|
|
|
|
def get_retro_hunt_completed_tasks():
|
|
return r_tracker.smembers('retro_hunts:completed')
|
|
|
|
## Change STATES ##
|
|
|
|
def get_retro_hunt_task_to_start():
|
|
task_uuid = r_tracker.spop('retro_hunts:pending')
|
|
if task_uuid:
|
|
retro_hunt = RetroHunt(task_uuid)
|
|
retro_hunt.run()
|
|
return task_uuid
|
|
|
|
## Metadata ##
|
|
|
|
def get_retro_hunt_metas():
|
|
tasks = []
|
|
for task_uuid in get_all_retro_hunt_tasks():
|
|
retro_hunt = RetroHunt(task_uuid)
|
|
tasks.append(retro_hunt.get_meta(options={'date', 'progress', 'nb_match', 'tags'}))
|
|
return tasks
|
|
|
|
## Objects ##
|
|
|
|
def is_obj_retro_hunted(obj_type, subtype, obj_id):
|
|
return r_tracker.exists(f'obj:retro_hunts:{obj_type}:{subtype}:{obj_id}')
|
|
|
|
def get_obj_retro_hunts(obj_type, subtype, obj_id):
|
|
return r_tracker.smembers(f'obj:retro_hunts:{obj_type}:{subtype}:{obj_id}')
|
|
|
|
def delete_obj_retro_hunts(obj_type, subtype, obj_id):
|
|
for retro_uuid in get_obj_retro_hunts(obj_type, subtype, obj_id):
|
|
retro_hunt = RetroHunt(retro_uuid)
|
|
retro_hunt.remove(obj_type, subtype, obj_id)
|
|
|
|
## API ##
|
|
def api_check_retro_hunt_task_uuid(task_uuid):
|
|
if not is_valid_uuid_v4(task_uuid):
|
|
return {"status": "error", "reason": "Invalid uuid"}, 400
|
|
retro_hunt = RetroHunt(task_uuid)
|
|
if not retro_hunt.exists():
|
|
return {"status": "error", "reason": "Unknown uuid"}, 404
|
|
return None
|
|
|
|
def api_pause_retro_hunt_task(task_uuid):
|
|
res = api_check_retro_hunt_task_uuid(task_uuid)
|
|
if res:
|
|
return res
|
|
retro_hunt = RetroHunt(task_uuid)
|
|
task_state = retro_hunt.get_state()
|
|
if task_state not in ['pending', 'running']:
|
|
return {"status": "error", "reason": f"Task {task_uuid} not paused, current state: {task_state}"}, 400
|
|
retro_hunt.pause()
|
|
return task_uuid, 200
|
|
|
|
def api_resume_retro_hunt_task(task_uuid):
|
|
res = api_check_retro_hunt_task_uuid(task_uuid)
|
|
if res:
|
|
return res
|
|
retro_hunt = RetroHunt(task_uuid)
|
|
if not retro_hunt.is_paused():
|
|
return {"status": "error",
|
|
"reason": f"Task {task_uuid} not paused, current state: {retro_hunt.get_state()}"}, 400
|
|
retro_hunt.resume()
|
|
return task_uuid, 200
|
|
|
|
def api_validate_rule_to_add(rule, rule_type):
|
|
if rule_type == 'yara_custom':
|
|
if not is_valid_yara_rule(rule):
|
|
return {"status": "error", "reason": "Invalid custom Yara Rule"}, 400
|
|
elif rule_type == 'yara_default':
|
|
if not is_valid_default_yara_rule(rule):
|
|
return {"status": "error", "reason": "The Yara Rule doesn't exist"}, 400
|
|
else:
|
|
return {"status": "error", "reason": "Incorrect type"}, 400
|
|
return {"status": "success", "rule": rule, "type": rule_type}, 200
|
|
|
|
def api_create_retro_hunt_task(dict_input, creator):
|
|
# # TODO: API: check mandatory arg
|
|
# # TODO: TIMEOUT
|
|
|
|
# timeout=30
|
|
rule = dict_input.get('rule', None)
|
|
if not rule:
|
|
return {"status": "error", "reason": "Retro Hunt Rule not provided"}, 400
|
|
task_type = dict_input.get('type', None)
|
|
if not task_type:
|
|
return {"status": "error", "reason": "type not provided"}, 400
|
|
|
|
# # TODO: limit
|
|
name = dict_input.get('name', '')
|
|
name = escape(name)
|
|
name = name[:60]
|
|
# # TODO: limit
|
|
description = dict_input.get('description', '')
|
|
description = escape(description)
|
|
description = description[:1000]
|
|
|
|
res = api_validate_rule_to_add(rule, task_type)
|
|
if res[1] != 200:
|
|
return res
|
|
|
|
tags = dict_input.get('tags', []) # TODO escape custom tags
|
|
mails = dict_input.get('mails', []) # TODO escape mails
|
|
res = verify_mail_list(mails)
|
|
if res:
|
|
return res
|
|
|
|
# Filters # TODO MOVE ME
|
|
filters = dict_input.get('filters', {})
|
|
if filters:
|
|
if filters.keys() == get_objects_retro_hunted():
|
|
filters = {}
|
|
for obj_type in filters:
|
|
if obj_type not in get_objects_retro_hunted():
|
|
return {"status": "error", "reason": "Invalid Tracker Object type"}, 400
|
|
|
|
for filter_name in filters[obj_type]:
|
|
if filter_name not in {'date_from', 'date_to', 'mimetypes', 'sources', 'subtypes'}:
|
|
return {"status": "error", "reason": "Invalid Filter"}, 400
|
|
elif filter_name == 'date_from':
|
|
if not Date.validate_str_date(filters[obj_type]['date_from']):
|
|
return {"status": "error", "reason": "Invalid date_from"}, 400
|
|
elif filter_name == 'date_to':
|
|
if not Date.validate_str_date(filters[obj_type]['date_from']):
|
|
return {"status": "error", "reason": "Invalid date_to"}, 400
|
|
elif filter_name == 'mimetypes': # TODO sanityze mimetypes
|
|
pass
|
|
elif filter_name == 'sources':
|
|
if obj_type == 'item':
|
|
res = item_basic.verify_sources_list(filters['item']['sources'])
|
|
if res:
|
|
return res
|
|
else:
|
|
return {"status": "error", "reason": "Invalid Filter sources"}, 400
|
|
elif filter_name == 'subtypes':
|
|
obj_subtypes = set(get_object_all_subtypes(obj_type))
|
|
for subtype in filters[obj_type]['subtypes']:
|
|
if subtype not in obj_subtypes:
|
|
return {"status": "error", "reason": "Invalid Tracker Object subtype"}, 400
|
|
|
|
if 'date_from' and 'date_to' in filters:
|
|
res = Date.api_validate_str_date_range(filters[obj_type]['date_from'], filters[obj_type]['date_to'])
|
|
if res:
|
|
return res
|
|
|
|
task_uuid = create_retro_hunt(name, task_type, rule, creator, description=description,
|
|
mails=mails, tags=tags, timeout=30, filters=filters)
|
|
return {'name': name, 'rule': rule, 'type': task_type, 'uuid': task_uuid}, 200
|
|
|
|
def api_delete_retro_hunt_task(task_uuid):
|
|
res = api_check_retro_hunt_task_uuid(task_uuid)
|
|
if res:
|
|
return res
|
|
retro_hunt = RetroHunt(task_uuid)
|
|
if retro_hunt.is_running() and retro_hunt.get_state() not in ['completed', 'paused']:
|
|
return {"status": "error", "reason": "You can't delete a running task"}, 400
|
|
else:
|
|
return retro_hunt.delete(), 200
|
|
|
|
################################################################################
|
|
################################################################################
|
|
################################################################################
|
|
################################################################################
|
|
|
|
#### DB FIX #### TODO
|
|
|
|
def _fix_db_custom_tags():
|
|
for tag in get_trackers_tags():
|
|
if not Tag.is_taxonomie_tag(tag) and not Tag.is_galaxy_tag(tag):
|
|
Tag.create_custom_tag(tag)
|
|
|
|
#### -- ####
|
|
|
|
|
|
# if __name__ == '__main__':
|
|
|
|
# _fix_db_custom_tags()
|
|
# fix_all_tracker_uuid_list()
|
|
# res = get_all_tracker_uuid()
|
|
# print(len(res))
|
|
|
|
# import Term
|
|
# Term.delete_term('5262ab6c-8784-4a55-b0ff-a471018414b4')
|
|
|
|
#fix_tracker_stats_per_day('5262ab6c-8784-4a55-b0ff-a471018414b4')
|
|
|
|
# tracker_uuid = '5262ab6c-8784-4a55-b0ff-a471018414b4'
|
|
# fix_tracker_item_link(tracker_uuid)
|
|
# res = get_item_all_trackers_uuid('archive/')
|
|
# print(res)
|
|
|
|
#res = is_valid_yara_rule('rule dummy { }')
|
|
|
|
# res = create_tracker('test', 'word', 'admin@admin.test', 1, [], [], None, sources=['crawled', 'pastebin.com', 'rt/pastebin.com'])
|
|
#res = create_tracker('circl\.lu', 'regex', 'admin@admin.test', 1, [], [], None, sources=['crawled','pastebin.com'])
|
|
#print(res)
|
|
|
|
#t_uuid = '1c2d35b0-9330-4feb-b454-da13007aa9f7'
|
|
#res = get_tracker_sources('ail-yara-rules/rules/crypto/certificate.yar', 'yara')
|
|
|
|
# sys.path.append(os.environ['AIL_BIN'])
|
|
# from packages import Term
|
|
# Term.delete_term('074ab4be-6049-45b5-a20e-8125a4e4f500')
|
|
|
|
|
|
#res = get_items_to_analyze('archive/pastebin.com_pro/2020/05/15', last='archive/pastebin.com_pro/2020/05/15/zkHEgqjQ.gz')
|
|
#get_retro_hunt_task_progress('0', nb_src_done=2)
|
|
|
|
#res = set_cache_retro_hunt_task_progress('0', 100)
|
|
#res = get_retro_hunt_task_nb_src_done('0', sources=['pastebin.com_pro', 'alerts/pastebin.com_pro', 'crawled'])
|
|
#print(res)
|
|
|
|
# sources = ['pastebin.com_pro', 'alerts/pastebin.com_pro', 'crawled']
|
|
# rule = 'custom-rules/4a8a3d04-f0b6-43ce-8e00-bdf47a8df241.yar'
|
|
# name = 'retro_hunt_test_1'
|
|
# description = 'circl retro hunt first test'
|
|
# tags = ['retro_circl', 'circl']
|
|
# creator = 'admin@admin.test'
|
|
# date_from = '20200610'
|
|
# date_to = '20210630'
|
|
|
|
#res = create_retro_hunt_task(name, rule, date_from, date_to, creator, sources=sources, tags=tags, description=description)
|
|
|
|
|
|
#get_retro_hunt_nb_item_by_day(['80b402ef-a8a9-4e97-adb6-e090edcfd571'], date_from=None, date_to=None, num_day=31)
|
|
|
|
#res = get_retro_hunt_nb_item_by_day(['c625f971-16e6-4331-82a7-b1e1b9efdec1'], date_from='20200610', date_to='20210630')
|
|
|
|
#res = delete_retro_hunt_task('598687b6-f765-4f8b-861a-09ad76d0ab34')
|
|
|
|
#print(res)
|