mirror of https://github.com/CIRCL/AIL-framework
chg: [Tracker] edit tracker
parent
e5ef95030c
commit
4fa320741c
|
@ -2,19 +2,78 @@
|
||||||
# -*-coding:UTF-8 -*
|
# -*-coding:UTF-8 -*
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
import redis
|
import redis
|
||||||
|
import uuid
|
||||||
import yara
|
import yara
|
||||||
|
import datetime
|
||||||
|
|
||||||
|
from flask import escape
|
||||||
|
|
||||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||||
import ConfigLoader
|
import ConfigLoader
|
||||||
#import item_basic
|
#import item_basic
|
||||||
|
|
||||||
config_loader = ConfigLoader.ConfigLoader()
|
config_loader = ConfigLoader.ConfigLoader()
|
||||||
|
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
|
||||||
r_serv_tracker = config_loader.get_redis_conn("ARDB_Tracker")
|
r_serv_tracker = config_loader.get_redis_conn("ARDB_Tracker")
|
||||||
config_loader = None
|
config_loader = None
|
||||||
|
|
||||||
|
email_regex = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}'
|
||||||
|
email_regex = re.compile(email_regex)
|
||||||
|
|
||||||
|
special_characters = set('[<>~!?@#$%^&*|()_-+={}":;,.\'\n\r\t]/\\')
|
||||||
|
special_characters.add('\\s')
|
||||||
|
|
||||||
|
###############
|
||||||
|
#### UTILS ####
|
||||||
|
def is_valid_uuid_v4(UUID):
|
||||||
|
if not UUID:
|
||||||
|
return False
|
||||||
|
UUID = UUID.replace('-', '')
|
||||||
|
try:
|
||||||
|
uuid_test = uuid.UUID(hex=UUID, version=4)
|
||||||
|
return uuid_test.hex == UUID
|
||||||
|
except:
|
||||||
|
return False
|
||||||
|
|
||||||
|
def is_valid_regex(tracker_regex):
|
||||||
|
try:
|
||||||
|
re.compile(tracker_regex)
|
||||||
|
return True
|
||||||
|
except:
|
||||||
|
return False
|
||||||
|
|
||||||
|
def is_valid_mail(email):
|
||||||
|
result = email_regex.match(email)
|
||||||
|
if result:
|
||||||
|
return True
|
||||||
|
else:
|
||||||
|
return False
|
||||||
|
|
||||||
|
def verify_mail_list(mail_list):
|
||||||
|
for mail in mail_list:
|
||||||
|
if not is_valid_mail(mail):
|
||||||
|
return ({'status': 'error', 'reason': 'Invalid email', 'value': mail}, 400)
|
||||||
|
return None
|
||||||
|
|
||||||
|
##-- UTILS --##
|
||||||
|
###############
|
||||||
|
|
||||||
|
def get_tracker_by_uuid(tracker_uuid):
|
||||||
|
return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'tracked')
|
||||||
|
|
||||||
|
def get_tracker_type(tracker_uuid):
|
||||||
|
return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'type')
|
||||||
|
|
||||||
|
def get_tracker_level(tracker_uuid):
|
||||||
|
return int(r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'level'))
|
||||||
|
|
||||||
|
def get_tracker_user_id(tracker_uuid):
|
||||||
|
return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'user_id')
|
||||||
|
|
||||||
def get_tracker_uuid_list(tracker, tracker_type):
|
def get_tracker_uuid_list(tracker, tracker_type):
|
||||||
return list(r_serv_tracker.smembers('all:tracker_uuid:{}:{}'.format(tracker_type, tracker)))
|
return list(r_serv_tracker.smembers('all:tracker_uuid:{}:{}'.format(tracker_type, tracker)))
|
||||||
|
|
||||||
|
@ -27,6 +86,51 @@ def get_tracker_mails(tracker_uuid):
|
||||||
def get_tracker_description(tracker_uuid):
|
def get_tracker_description(tracker_uuid):
|
||||||
return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'description')
|
return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'description')
|
||||||
|
|
||||||
|
def get_tracker_first_seen(tracker_uuid):
|
||||||
|
res = r_serv_tracker.zrange('tracker:stat:{}'.format(tracker_uuid), 0, 0)
|
||||||
|
if res:
|
||||||
|
return res[0]
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def get_tracker_last_seen(tracker_uuid):
|
||||||
|
res = r_serv_tracker.zrevrange('tracker:stat:{}'.format(tracker_uuid), 0, 0)
|
||||||
|
if res:
|
||||||
|
return res[0]
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def get_tracker_metedata(tracker_uuid, user_id=False, description=False, level=False, tags=False, mails=False, sparkline=False):
|
||||||
|
dict_uuid = {}
|
||||||
|
dict_uuid['tracker'] = get_tracker_by_uuid(tracker_uuid)
|
||||||
|
dict_uuid['type'] = get_tracker_type(tracker_uuid)
|
||||||
|
dict_uuid['date'] = r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'date')
|
||||||
|
dict_uuid['description'] = get_tracker_description(tracker_uuid)
|
||||||
|
dict_uuid['first_seen'] = get_tracker_first_seen(tracker_uuid)
|
||||||
|
dict_uuid['last_seen'] = get_tracker_last_seen(tracker_uuid)
|
||||||
|
if user_id:
|
||||||
|
dict_uuid['user_id'] = get_tracker_user_id(tracker_uuid)
|
||||||
|
if level:
|
||||||
|
dict_uuid['level'] = get_tracker_level(tracker_uuid)
|
||||||
|
if mails:
|
||||||
|
dict_uuid['mails'] = get_tracker_mails(tracker_uuid)
|
||||||
|
if tags:
|
||||||
|
dict_uuid['tags'] = get_tracker_tags(tracker_uuid)
|
||||||
|
if sparkline:
|
||||||
|
dict_uuid['sparkline'] = get_tracker_sparkline(tracker_uuid)
|
||||||
|
dict_uuid['uuid'] = tracker_uuid
|
||||||
|
return dict_uuid
|
||||||
|
|
||||||
|
def get_tracker_sparkline(tracker_uuid, num_day=6):
|
||||||
|
date_range_sparkline = Date.get_date_range(num_day)
|
||||||
|
sparklines_value = []
|
||||||
|
for date_day in date_range_sparkline:
|
||||||
|
nb_seen_this_day = r_serv_tracker.scard('tracker:item:{}:{}'.format(tracker_uuid, date_day))
|
||||||
|
if nb_seen_this_day is None:
|
||||||
|
nb_seen_this_day = 0
|
||||||
|
sparklines_value.append(int(nb_seen_this_day))
|
||||||
|
return sparklines_value
|
||||||
|
|
||||||
def add_tracked_item(tracker_uuid, item_id, item_date):
|
def add_tracked_item(tracker_uuid, item_id, item_date):
|
||||||
# track item
|
# track item
|
||||||
r_serv_tracker.sadd('tracker:item:{}:{}'.format(tracker_uuid, item_date), item_id)
|
r_serv_tracker.sadd('tracker:item:{}:{}'.format(tracker_uuid, item_date), item_id)
|
||||||
|
@ -46,6 +150,234 @@ def get_tracker_last_updated_by_type(tracker_type):
|
||||||
epoch_update = 0
|
epoch_update = 0
|
||||||
return float(epoch_update)
|
return float(epoch_update)
|
||||||
|
|
||||||
|
######################
|
||||||
|
#### TRACKERS ACL ####
|
||||||
|
|
||||||
|
# # TODO: use new package => duplicate fct
|
||||||
|
def is_in_role(user_id, role):
|
||||||
|
if r_serv_db.sismember('user_role:{}'.format(role), user_id):
|
||||||
|
return True
|
||||||
|
else:
|
||||||
|
return False
|
||||||
|
|
||||||
|
def is_tracker_in_global_level(tracker, tracker_type):
|
||||||
|
res = r_serv_tracker.smembers('all:tracker_uuid:{}:{}'.format(tracker_type, tracker))
|
||||||
|
if res:
|
||||||
|
for elem_uuid in res:
|
||||||
|
if r_serv_tracker.hget('tracker:{}'.format(elem_uuid), 'level')=='1':
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def is_tracker_in_user_level(tracker, tracker_type, user_id):
|
||||||
|
res = r_serv_tracker.smembers('user:tracker:{}'.format(user_id))
|
||||||
|
if res:
|
||||||
|
for elem_uuid in res:
|
||||||
|
if r_serv_tracker.hget('tracker:{}'.format(elem_uuid), 'tracked')== tracker:
|
||||||
|
if r_serv_tracker.hget('tracker:{}'.format(elem_uuid), 'type')== tracker_type:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def api_is_allowed_to_edit_tracker(tracker_uuid, user_id):
|
||||||
|
if not is_valid_uuid_v4(tracker_uuid):
|
||||||
|
return ({"status": "error", "reason": "Invalid uuid"}, 400)
|
||||||
|
tracker_creator = r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'user_id')
|
||||||
|
if not tracker_creator:
|
||||||
|
return ({"status": "error", "reason": "Unknown uuid"}, 404)
|
||||||
|
if not is_in_role(user_id, 'admin') or user_id != tracker_creator:
|
||||||
|
return ({"status": "error", "reason": "Access Denied"}, 403)
|
||||||
|
return ({"uuid": tracker_uuid}, 200)
|
||||||
|
|
||||||
|
|
||||||
|
##-- ACL --##
|
||||||
|
|
||||||
|
#### CREATE TRACKER ####
|
||||||
|
def api_validate_tracker_to_add(tracker , tracker_type, nb_words=1):
|
||||||
|
if tracker_type=='regex':
|
||||||
|
if not is_valid_regex(tracker):
|
||||||
|
return ({"status": "error", "reason": "Invalid regex"}, 400)
|
||||||
|
elif tracker_type=='word' or tracker_type=='set':
|
||||||
|
# force lowercase
|
||||||
|
tracker = tracker.lower()
|
||||||
|
word_set = set(tracker)
|
||||||
|
set_inter = word_set.intersection(special_characters)
|
||||||
|
if set_inter:
|
||||||
|
return ({"status": "error", "reason": f'special character(s) not allowed: {set_inter}', "message": "Please use a python regex or remove all special characters"}, 400)
|
||||||
|
words = tracker.split()
|
||||||
|
# not a word
|
||||||
|
if tracker_type=='word' and len(words)>1:
|
||||||
|
tracker_type = 'set'
|
||||||
|
|
||||||
|
# ouput format: tracker1,tracker2,tracker3;2
|
||||||
|
if tracker_type=='set':
|
||||||
|
try:
|
||||||
|
nb_words = int(nb_words)
|
||||||
|
except:
|
||||||
|
nb_words = 1
|
||||||
|
if nb_words==0:
|
||||||
|
nb_words = 1
|
||||||
|
|
||||||
|
words_set = set(words)
|
||||||
|
words_set = sorted(words_set)
|
||||||
|
|
||||||
|
if nb_words > len(words_set):
|
||||||
|
nb_words = len(words_set)
|
||||||
|
|
||||||
|
tracker = ",".join(words_set)
|
||||||
|
tracker = "{};{}".format(tracker, nb_words)
|
||||||
|
|
||||||
|
elif tracker_type=='yara_custom':
|
||||||
|
if not is_valid_yara_rule(tracker):
|
||||||
|
return ({"status": "error", "reason": "Invalid custom Yara Rule"}, 400)
|
||||||
|
elif tracker_type=='yara_default':
|
||||||
|
if not is_valid_default_yara_rule(tracker):
|
||||||
|
return ({"status": "error", "reason": "The Yara Rule doesn't exist"}, 400)
|
||||||
|
else:
|
||||||
|
return ({"status": "error", "reason": "Incorrect type"}, 400)
|
||||||
|
return ({"status": "success", "tracker": tracker, "type": tracker_type}, 200)
|
||||||
|
|
||||||
|
def create_tracker(tracker, tracker_type, user_id, level, tags, mails, description, dashboard=0, tracker_uuid=None):
|
||||||
|
# edit tracker
|
||||||
|
if tracker_uuid:
|
||||||
|
edit_tracker = True
|
||||||
|
# check if type changed
|
||||||
|
old_type = get_tracker_type(tracker_uuid)
|
||||||
|
old_tracker = get_tracker_by_uuid(tracker_uuid)
|
||||||
|
old_level = get_tracker_level(tracker_uuid)
|
||||||
|
tracker_user_id = get_tracker_user_id(tracker_uuid)
|
||||||
|
|
||||||
|
# Create new tracker
|
||||||
|
else:
|
||||||
|
edit_tracker = False
|
||||||
|
# generate tracker uuid
|
||||||
|
tracker_uuid = str(uuid.uuid4())
|
||||||
|
old_type = None
|
||||||
|
old_tracker = None
|
||||||
|
|
||||||
|
# YARA
|
||||||
|
if tracker_type == 'yara_custom' or tracker_type == 'yara_default':
|
||||||
|
# delete yara rule
|
||||||
|
if tracker_type == 'yara_default' and old_type == 'yara':
|
||||||
|
if not is_default_yara_rule(old_tracker):
|
||||||
|
filepath = get_yara_rule_file_by_tracker_name(old_tracker)
|
||||||
|
if filepath:
|
||||||
|
os.remove(filepath)
|
||||||
|
tracker = save_yara_rule(tracker_type, tracker, tracker_uuid=tracker_uuid)
|
||||||
|
tracker_type = 'yara'
|
||||||
|
|
||||||
|
# create metadata
|
||||||
|
r_serv_tracker.hset('tracker:{}'.format(tracker_uuid), 'tracked', tracker)
|
||||||
|
r_serv_tracker.hset('tracker:{}'.format(tracker_uuid), 'type', tracker_type)
|
||||||
|
r_serv_tracker.hset('tracker:{}'.format(tracker_uuid), 'date', datetime.date.today().strftime("%Y%m%d"))
|
||||||
|
r_serv_tracker.hset('tracker:{}'.format(tracker_uuid), 'level', level)
|
||||||
|
r_serv_tracker.hset('tracker:{}'.format(tracker_uuid), 'dashboard', dashboard)
|
||||||
|
if not edit_tracker:
|
||||||
|
r_serv_tracker.hset('tracker:{}'.format(tracker_uuid), 'user_id', user_id)
|
||||||
|
|
||||||
|
if description:
|
||||||
|
r_serv_tracker.hset('tracker:{}'.format(tracker_uuid), 'description', description)
|
||||||
|
|
||||||
|
# type change
|
||||||
|
if edit_tracker:
|
||||||
|
r_serv_tracker.srem('all:tracker:{}'.format(old_type), old_tracker)
|
||||||
|
r_serv_tracker.srem('all:tracker_uuid:{}:{}'.format(old_type, old_tracker), tracker_uuid)
|
||||||
|
if level != old_level:
|
||||||
|
if level == 0:
|
||||||
|
r_serv_tracker.srem('global:tracker', tracker_uuid)
|
||||||
|
elif level == 1:
|
||||||
|
r_serv_tracker.srem('user:tracker:{}'.format(tracker_user_id), tracker_uuid)
|
||||||
|
if tracker_type != old_type:
|
||||||
|
if old_level == 0:
|
||||||
|
r_serv_tracker.srem('user:tracker:{}:{}'.format(tracker_user_id, old_type), tracker_uuid)
|
||||||
|
elif old_level == 1:
|
||||||
|
r_serv_tracker.srem('global:tracker:{}'.format(old_type), tracker_uuid)
|
||||||
|
if old_type=='yara':
|
||||||
|
if not is_default_yara_rule(old_tracker):
|
||||||
|
filepath = get_yara_rule_file_by_tracker_name(old_tracker)
|
||||||
|
if filepath:
|
||||||
|
os.remove(filepath)
|
||||||
|
|
||||||
|
# create all tracker set
|
||||||
|
r_serv_tracker.sadd('all:tracker:{}'.format(tracker_type), tracker)
|
||||||
|
|
||||||
|
# create tracker - uuid map
|
||||||
|
r_serv_tracker.sadd('all:tracker_uuid:{}:{}'.format(tracker_type, tracker), tracker_uuid)
|
||||||
|
|
||||||
|
# add display level set
|
||||||
|
if level == 0: # user only
|
||||||
|
r_serv_tracker.sadd('user:tracker:{}'.format(user_id), tracker_uuid)
|
||||||
|
r_serv_tracker.sadd('user:tracker:{}:{}'.format(user_id, tracker_type), tracker_uuid)
|
||||||
|
elif level == 1: # global
|
||||||
|
r_serv_tracker.sadd('global:tracker', tracker_uuid)
|
||||||
|
r_serv_tracker.sadd('global:tracker:{}'.format(tracker_type), tracker_uuid)
|
||||||
|
|
||||||
|
# create tracker tags list
|
||||||
|
for tag in tags:
|
||||||
|
r_serv_tracker.sadd('tracker:tags:{}'.format(tracker_uuid), escape(tag) )
|
||||||
|
|
||||||
|
# create tracker tags mail notification list
|
||||||
|
for mail in mails:
|
||||||
|
r_serv_tracker.sadd('tracker:mail:{}'.format(tracker_uuid), escape(mail) )
|
||||||
|
|
||||||
|
# toggle refresh module tracker list/set
|
||||||
|
r_serv_tracker.set('tracker:refresh:{}'.format(tracker_type), time.time())
|
||||||
|
if tracker_type != old_type: # toggle old type refresh
|
||||||
|
r_serv_tracker.set('tracker:refresh:{}'.format(old_type), time.time())
|
||||||
|
return tracker_uuid
|
||||||
|
|
||||||
|
def api_add_tracker(dict_input, user_id):
|
||||||
|
tracker = dict_input.get('tracker', None)
|
||||||
|
if not tracker:
|
||||||
|
return ({"status": "error", "reason": "Tracker not provided"}, 400)
|
||||||
|
tracker_type = dict_input.get('type', None)
|
||||||
|
if not tracker_type:
|
||||||
|
return ({"status": "error", "reason": "Tracker type not provided"}, 400)
|
||||||
|
nb_words = dict_input.get('nb_words', 1)
|
||||||
|
description = dict_input.get('description', '')
|
||||||
|
description = escape(description)
|
||||||
|
|
||||||
|
res = api_validate_tracker_to_add(tracker , tracker_type, nb_words=nb_words)
|
||||||
|
if res[1]!=200:
|
||||||
|
return res
|
||||||
|
tracker = res[0]['tracker']
|
||||||
|
tracker_type = res[0]['type']
|
||||||
|
|
||||||
|
tags = dict_input.get('tags', [])
|
||||||
|
mails = dict_input.get('mails', [])
|
||||||
|
res = verify_mail_list(mails)
|
||||||
|
if res:
|
||||||
|
return res
|
||||||
|
|
||||||
|
## TODO: add dashboard key
|
||||||
|
level = dict_input.get('level', 1)
|
||||||
|
try:
|
||||||
|
level = int(level)
|
||||||
|
if level not in range(0, 1):
|
||||||
|
level = 1
|
||||||
|
except:
|
||||||
|
level = 1
|
||||||
|
|
||||||
|
tracker_uuid = dict_input.get('uuid', None)
|
||||||
|
# check edit ACL
|
||||||
|
if tracker_uuid:
|
||||||
|
res = api_is_allowed_to_edit_tracker(tracker_uuid, user_id)
|
||||||
|
if res[1] != 200:
|
||||||
|
return res
|
||||||
|
else:
|
||||||
|
# check if tracker already tracked in global
|
||||||
|
if level==1:
|
||||||
|
if is_tracker_in_global_level(tracker, tracker_type) and not tracker_uuid:
|
||||||
|
return ({"status": "error", "reason": "Tracker already exist"}, 409)
|
||||||
|
else:
|
||||||
|
if is_tracker_in_user_level(tracker, tracker_type, user_id) and not tracker_uuid:
|
||||||
|
return ({"status": "error", "reason": "Tracker already exist"}, 409)
|
||||||
|
|
||||||
|
tracker_uuid = create_tracker(tracker , tracker_type, user_id, level, tags, mails, description, tracker_uuid=tracker_uuid)
|
||||||
|
|
||||||
|
return ({'tracker': tracker, 'type': tracker_type, 'uuid': tracker_uuid}, 200)
|
||||||
|
|
||||||
|
##-- CREATE TRACKER --##
|
||||||
|
|
||||||
|
##############
|
||||||
#### YARA ####
|
#### YARA ####
|
||||||
def get_yara_rules_dir():
|
def get_yara_rules_dir():
|
||||||
return os.path.join(os.environ['AIL_BIN'], 'trackers', 'yara')
|
return os.path.join(os.environ['AIL_BIN'], 'trackers', 'yara')
|
||||||
|
@ -99,15 +431,32 @@ def is_valid_yara_rule(yara_rule):
|
||||||
except:
|
except:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def is_valid_default_yara_rule(yara_rule):
|
def is_default_yara_rule(tracked_yara_name):
|
||||||
|
yara_dir = get_yara_rules_dir()
|
||||||
|
filename = os.path.join(yara_dir, tracked_yara_name)
|
||||||
|
filename = os.path.realpath(filename)
|
||||||
|
try:
|
||||||
|
if tracked_yara_name.split('/')[0] == 'custom-rules':
|
||||||
|
return False
|
||||||
|
except:
|
||||||
|
return False
|
||||||
|
if not os.path.commonprefix([filename, yara_dir]) == yara_dir:
|
||||||
|
return False
|
||||||
|
else:
|
||||||
|
if os.path.isfile(filename):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def is_valid_default_yara_rule(yara_rule, verbose=True):
|
||||||
yara_dir = get_yara_rules_default_dir()
|
yara_dir = get_yara_rules_default_dir()
|
||||||
filename = os.path.join(yara_dir, yara_rule)
|
filename = os.path.join(yara_dir, yara_rule)
|
||||||
filename = os.path.realpath(filename)
|
filename = os.path.realpath(filename)
|
||||||
# incorrect filename
|
# incorrect filename
|
||||||
if not os.path.commonprefix([filename, yara_dir]) == yara_dir:
|
if not os.path.commonprefix([filename, yara_dir]) == yara_dir:
|
||||||
print('error: file transversal')
|
if verbose:
|
||||||
print(yara_dir)
|
print('error: file transversal')
|
||||||
print(filename)
|
print(yara_dir)
|
||||||
|
print(filename)
|
||||||
return False
|
return False
|
||||||
else:
|
else:
|
||||||
if os.path.isfile(filename):
|
if os.path.isfile(filename):
|
||||||
|
@ -126,6 +475,17 @@ def save_yara_rule(yara_rule_type, yara_rule, tracker_uuid=None):
|
||||||
filename = os.path.join('ail-yara-rules', 'rules', yara_rule)
|
filename = os.path.join('ail-yara-rules', 'rules', yara_rule)
|
||||||
return filename
|
return filename
|
||||||
|
|
||||||
|
def get_yara_rule_file_by_tracker_name(tracked_yara_name):
|
||||||
|
yara_dir = get_yara_rules_dir()
|
||||||
|
filename = os.path.join(yara_dir, tracked_yara_name)
|
||||||
|
filename = os.path.realpath(filename)
|
||||||
|
if not os.path.commonprefix([filename, yara_dir]) == yara_dir:
|
||||||
|
print('error: file transversal')
|
||||||
|
print(yara_dir)
|
||||||
|
print(filename)
|
||||||
|
return None
|
||||||
|
return filename
|
||||||
|
|
||||||
def get_yara_rule_content(yara_rule):
|
def get_yara_rule_content(yara_rule):
|
||||||
yara_dir = get_yara_rules_dir()
|
yara_dir = get_yara_rules_dir()
|
||||||
filename = os.path.join(yara_dir, yara_rule)
|
filename = os.path.join(yara_dir, yara_rule)
|
||||||
|
@ -157,7 +517,6 @@ def api_get_default_rule_content(default_yara_rule):
|
||||||
|
|
||||||
##-- YARA --##
|
##-- YARA --##
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
res = is_valid_yara_rule('rule dummy { }')
|
res = is_valid_yara_rule('rule dummy { }')
|
||||||
print(res)
|
print(res)
|
||||||
|
|
|
@ -38,6 +38,8 @@ tokenizer = RegexpTokenizer('[\&\~\:\;\,\.\(\)\{\}\|\[\]\\\\/\-/\=\'\"\%\$\?\@\+
|
||||||
gaps=True, discard_empty=True)
|
gaps=True, discard_empty=True)
|
||||||
|
|
||||||
def is_valid_uuid_v4(UUID):
|
def is_valid_uuid_v4(UUID):
|
||||||
|
if not UUID:
|
||||||
|
return False
|
||||||
UUID = UUID.replace('-', '')
|
UUID = UUID.replace('-', '')
|
||||||
try:
|
try:
|
||||||
uuid_test = uuid.UUID(hex=UUID, version=4)
|
uuid_test = uuid.UUID(hex=UUID, version=4)
|
||||||
|
@ -215,11 +217,12 @@ def parse_tracked_term_to_add(term , term_type, nb_words=1):
|
||||||
words_set = set(words)
|
words_set = set(words)
|
||||||
words_set = sorted(words_set)
|
words_set = sorted(words_set)
|
||||||
|
|
||||||
|
if nb_words > len(words_set):
|
||||||
|
nb_words = len(words_set)
|
||||||
|
|
||||||
term = ",".join(words_set)
|
term = ",".join(words_set)
|
||||||
term = "{};{}".format(term, nb_words)
|
term = "{};{}".format(term, nb_words)
|
||||||
|
|
||||||
if nb_words > len(words_set):
|
|
||||||
nb_words = len(words_set)
|
|
||||||
elif term_type=='yara_custom':
|
elif term_type=='yara_custom':
|
||||||
if not Tracker.is_valid_yara_rule(term):
|
if not Tracker.is_valid_yara_rule(term):
|
||||||
return ({"status": "error", "reason": "Invalid custom Yara Rule"}, 400)
|
return ({"status": "error", "reason": "Invalid custom Yara Rule"}, 400)
|
||||||
|
@ -322,8 +325,11 @@ def delete_term(term_uuid):
|
||||||
r_serv_term.delete('tracker:stat:{}'.format(term_uuid))
|
r_serv_term.delete('tracker:stat:{}'.format(term_uuid))
|
||||||
|
|
||||||
if term_type == 'yara':
|
if term_type == 'yara':
|
||||||
# # TODO:
|
# delete custom rule
|
||||||
pass
|
if not Tracker.is_default_yara_rule(term):
|
||||||
|
filepath = Tracker.get_yara_rule_file_by_tracker_name(term)
|
||||||
|
if filepath:
|
||||||
|
os.remove(filepath)
|
||||||
|
|
||||||
def replace_tracker_description(term_uuid, description):
|
def replace_tracker_description(term_uuid, description):
|
||||||
description = escape(description)
|
description = escape(description)
|
||||||
|
|
|
@ -93,8 +93,9 @@ def tracked_menu_yara():
|
||||||
@login_analyst
|
@login_analyst
|
||||||
def add_tracked_menu():
|
def add_tracked_menu():
|
||||||
if request.method == 'POST':
|
if request.method == 'POST':
|
||||||
term = request.form.get("term")
|
tracker = request.form.get("tracker")
|
||||||
term_type = request.form.get("tracker_type")
|
tracker_uuid = request.form.get("tracker_uuid")
|
||||||
|
tracker_type = request.form.get("tracker_type")
|
||||||
nb_words = request.form.get("nb_word", 1)
|
nb_words = request.form.get("nb_word", 1)
|
||||||
description = request.form.get("description", '')
|
description = request.form.get("description", '')
|
||||||
level = request.form.get("level", 0)
|
level = request.form.get("level", 0)
|
||||||
|
@ -102,15 +103,15 @@ def add_tracked_menu():
|
||||||
mails = request.form.get("mails", [])
|
mails = request.form.get("mails", [])
|
||||||
|
|
||||||
# YARA #
|
# YARA #
|
||||||
if term_type == 'yara':
|
if tracker_type == 'yara':
|
||||||
yara_default_rule = request.form.get("yara_default_rule")
|
yara_default_rule = request.form.get("yara_default_rule")
|
||||||
yara_custom_rule = request.form.get("yara_custom_rule")
|
yara_custom_rule = request.form.get("yara_custom_rule")
|
||||||
if yara_custom_rule:
|
if yara_custom_rule:
|
||||||
term = yara_custom_rule
|
tracker = yara_custom_rule
|
||||||
term_type='yara_custom'
|
tracker_type='yara_custom'
|
||||||
else:
|
else:
|
||||||
term = yara_default_rule
|
tracker = yara_default_rule
|
||||||
term_type='yara_default'
|
tracker_type='yara_default'
|
||||||
# #
|
# #
|
||||||
|
|
||||||
if level == 'on':
|
if level == 'on':
|
||||||
|
@ -121,17 +122,58 @@ def add_tracked_menu():
|
||||||
if tags:
|
if tags:
|
||||||
tags = tags.split()
|
tags = tags.split()
|
||||||
|
|
||||||
input_dict = {"term": term, "type": term_type, "nb_words": nb_words, "tags": tags, "mails": mails, "level": level, "description": description}
|
input_dict = {"tracker": tracker, "type": tracker_type, "nb_words": nb_words, "tags": tags, "mails": mails, "level": level, "description": description}
|
||||||
user_id = current_user.get_id()
|
user_id = current_user.get_id()
|
||||||
res = Term.parse_json_term_to_add(input_dict, user_id)
|
# edit tracker
|
||||||
|
if tracker_uuid:
|
||||||
|
input_dict['uuid'] = tracker_uuid
|
||||||
|
res = Tracker.api_add_tracker(input_dict, user_id)
|
||||||
if res[1] == 200:
|
if res[1] == 200:
|
||||||
return redirect(url_for('hunter.tracked_menu'))
|
return redirect(url_for('hunter.tracked_menu'))
|
||||||
else:
|
else:
|
||||||
## TODO: use modal
|
## TODO: use modal
|
||||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
else:
|
else:
|
||||||
all_yara_files = Tracker.get_all_default_yara_files()
|
return render_template("edit_tracker.html", all_yara_files=Tracker.get_all_default_yara_files())
|
||||||
return render_template("Add_tracker.html", all_yara_files=all_yara_files)
|
|
||||||
|
@hunter.route("/tracker/edit", methods=['GET', 'POST'])
|
||||||
|
@login_required
|
||||||
|
@login_analyst
|
||||||
|
def edit_tracked_menu():
|
||||||
|
user_id = current_user.get_id()
|
||||||
|
tracker_uuid = request.args.get('uuid', None)
|
||||||
|
|
||||||
|
res = Term.check_term_uuid_valid_access(tracker_uuid, user_id) # check if is author or admin
|
||||||
|
if res: # invalid access
|
||||||
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
|
||||||
|
dict_tracker = Tracker.get_tracker_metedata(tracker_uuid, user_id=True, level=True, description=True, tags=True, mails=True)
|
||||||
|
dict_tracker['tags'] = ' '.join(dict_tracker['tags'])
|
||||||
|
dict_tracker['mails'] = ' '.join(dict_tracker['mails'])
|
||||||
|
|
||||||
|
if dict_tracker['type'] == 'set':
|
||||||
|
dict_tracker['tracker'], dict_tracker['nb_words'] = dict_tracker['tracker'].split(';')
|
||||||
|
dict_tracker['tracker'] = dict_tracker['tracker'].replace(',', ' ')
|
||||||
|
elif dict_tracker['type'] == 'yara': #is_valid_default_yara_rule
|
||||||
|
if Tracker.is_default_yara_rule(dict_tracker['tracker']):
|
||||||
|
dict_tracker['yara_file'] = dict_tracker['tracker'].split('/')
|
||||||
|
dict_tracker['yara_file'] = dict_tracker['yara_file'][-2] + '/' + dict_tracker['yara_file'][-1]
|
||||||
|
dict_tracker['content'] = None
|
||||||
|
else:
|
||||||
|
dict_tracker['yara_file'] = None
|
||||||
|
dict_tracker['content'] = Tracker.get_yara_rule_content(dict_tracker['tracker'])
|
||||||
|
|
||||||
|
return render_template("edit_tracker.html", dict_tracker=dict_tracker,
|
||||||
|
all_yara_files=Tracker.get_all_default_yara_files())
|
||||||
|
|
||||||
|
## TO EDIT
|
||||||
|
# word
|
||||||
|
# set of word + nb words
|
||||||
|
# regex
|
||||||
|
# yara custum
|
||||||
|
# yara default ???? => allow edit ?
|
||||||
|
|
||||||
|
#### EDIT SHow Trackers ??????????????????????????????????????????????????
|
||||||
|
|
||||||
@hunter.route("/tracker/show_tracker")
|
@hunter.route("/tracker/show_tracker")
|
||||||
@login_required
|
@login_required
|
||||||
|
|
|
@ -27,39 +27,41 @@
|
||||||
|
|
||||||
<div class="col-12 col-lg-10" id="core_content">
|
<div class="col-12 col-lg-10" id="core_content">
|
||||||
|
|
||||||
<div class="card mb-3 mt-1">
|
<div class="card my-3">
|
||||||
<div class="card-header">
|
<div class="card-header bg-dark text-white">
|
||||||
<h5 class="card-title">Create a new tracker</h5>
|
<h5 class="card-title">Edit a Tracker</h5>
|
||||||
</div>
|
</div>
|
||||||
<div class="card-body">
|
<div class="card-body">
|
||||||
<p class="card-text">Select a tracker type.</p>
|
|
||||||
|
|
||||||
<form action="{{ url_for('hunter.add_tracked_menu') }}" method='post'>
|
<form action="{{ url_for('hunter.add_tracked_menu') }}" method='post'>
|
||||||
|
{%if dict_tracker%}
|
||||||
|
<input id="tracker_uuid" name="tracker_uuid" class="form-control" type="text" value="{{dict_tracker['uuid']}}" hidden>
|
||||||
|
{%endif%}
|
||||||
|
|
||||||
<div class="row">
|
<div class="row">
|
||||||
<div class="col-12 col-xl-9">
|
<div class="col-12 col-xl-9">
|
||||||
<div class="input-group mb-2 mr-sm-2">
|
<div class="input-group mb-2 mr-sm-2">
|
||||||
<div class="input-group-prepend">
|
<div class="input-group-prepend">
|
||||||
<div class="input-group-text"><i class="fas fa-tag"></i></div>
|
<div class="input-group-text bg-danger text-white"><i class="fas fa-tag"></i></div>
|
||||||
</div>
|
</div>
|
||||||
<input id="tags" name="tags" class="form-control" placeholder="Tags (optional, space separated)" type="text">
|
<input id="tags" name="tags" class="form-control" placeholder="Tags (optional, space separated)" type="text" {%if dict_tracker%}{%if dict_tracker['tags']%}value="{{dict_tracker['tags']}}"{%endif%}{%endif%}>
|
||||||
</div>
|
</div>
|
||||||
<div class="input-group mb-2 mr-sm-2">
|
<div class="input-group mb-2 mr-sm-2">
|
||||||
<div class="input-group-prepend">
|
<div class="input-group-prepend">
|
||||||
<div class="input-group-text"><i class="fas fa-at"></i></div>
|
<div class="input-group-text bg-secondary text-white"><i class="fas fa-at"></i></div>
|
||||||
</div>
|
</div>
|
||||||
<input id="mails" name="mails" class="form-control" placeholder="E-Mails Notification (optional, space separated)" type="text">
|
<input id="mails" name="mails" class="form-control" placeholder="E-Mails Notification (optional, space separated)" type="text" {%if dict_tracker%}{%if dict_tracker['mails']%}value="{{dict_tracker['mails']}}"{%endif%}{%endif%}>
|
||||||
</div>
|
</div>
|
||||||
<div class="input-group mb-2 mr-sm-2">
|
<div class="input-group mb-2 mr-sm-2">
|
||||||
<div class="input-group-prepend">
|
<div class="input-group-prepend">
|
||||||
<div class="input-group-text"><i class="fas fa-pencil-alt"></i></div>
|
<div class="input-group-text bg-info text-white"><i class="fas fa-pencil-alt"></i></div>
|
||||||
</div>
|
</div>
|
||||||
<input id="description" name="description" class="form-control" placeholder="Tracker Description (optional)" type="text">
|
<input id="description" name="description" class="form-control" placeholder="Tracker Description (optional)" type="text" {%if dict_tracker%}{%if dict_tracker['description']%}value="{{dict_tracker['description']}}"{%endif%}{%endif%}>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="col-12 col-xl-3">
|
<div class="col-12 col-xl-3">
|
||||||
<div class="custom-control custom-switch mt-1">
|
<div class="custom-control custom-switch mt-1">
|
||||||
<input class="custom-control-input" type="checkbox" name="level" id="id_level" checked>
|
<input class="custom-control-input" type="checkbox" name="level" id="id_level" {%if dict_tracker%}{%if dict_tracker['level']==1%}checked{%endif%}{%else%}checked{%endif%}>
|
||||||
<label class="custom-control-label" for="id_level">
|
<label class="custom-control-label" for="id_level">
|
||||||
<i class="fas fa-users"></i> Show tracker to all Users
|
<i class="fas fa-users"></i> Show tracker to all Users
|
||||||
</label>
|
</label>
|
||||||
|
@ -68,6 +70,7 @@
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<hr>
|
<hr>
|
||||||
|
<h4>Tracker Type:</h4>
|
||||||
|
|
||||||
<select id="tracker_type" name="tracker_type" class="custom-select w-25 mb-3">
|
<select id="tracker_type" name="tracker_type" class="custom-select w-25 mb-3">
|
||||||
<option disabled selected value> -- Select a tracker type -- </option>
|
<option disabled selected value> -- Select a tracker type -- </option>
|
||||||
|
@ -81,10 +84,10 @@
|
||||||
|
|
||||||
<div class="row" id="simple_input">
|
<div class="row" id="simple_input">
|
||||||
<div class="col-12 col-lg-10">
|
<div class="col-12 col-lg-10">
|
||||||
<input id="term" name="term" class="form-control" placeholder="Terms to track (space separated)" type="text">
|
<input id="tracker" name="tracker" class="form-control" placeholder="Terms to track (space separated)" type="text" {%if dict_tracker%}{%if dict_tracker['tracker']!='yara'%}value="{{dict_tracker['tracker']}}"{%endif%}{%endif%}>
|
||||||
</div>
|
</div>
|
||||||
<div class="col-12 col-lg-2">
|
<div class="col-12 col-lg-2">
|
||||||
<input type="number" id="nb_word" name="nb_word" name="quantity" min="1" placeholder="Nb of keywords">
|
<input type="number" id="nb_word" name="nb_word" name="quantity" min="1" placeholder="Nb of keywords" {%if dict_tracker%}{%if dict_tracker['nb_words']%}value="{{dict_tracker['nb_words']}}"{%endif%}{%endif%}>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
@ -92,11 +95,12 @@
|
||||||
<div class="mb-2" id="yara_rule">
|
<div class="mb-2" id="yara_rule">
|
||||||
<div class="" id="yara_default_rules">
|
<div class="" id="yara_default_rules">
|
||||||
|
|
||||||
<select class="custom-select w-100 mb-3" name="yara_default_rule" onchange="get_default_rule_content(this);">
|
<h6>Default YARA rules:</h6>
|
||||||
|
<select class="custom-select w-100 mb-3" id="yara_default_rule" name="yara_default_rule" onchange="get_default_rule_content(this);">
|
||||||
<option selected>Select a default rule</option>
|
<option selected>Select a default rule</option>
|
||||||
{% for yara_types in all_yara_files %}
|
{% for yara_types in all_yara_files %}
|
||||||
{% for yara_file in all_yara_files[yara_types] %}
|
{% for yara_file_name in all_yara_files[yara_types] %}
|
||||||
<option value="{{yara_types}}/{{yara_file}}">{{yara_types}} - {{yara_file}}</option>
|
<option value="{{yara_types}}/{{yara_file_name}}">{{yara_types}} - {{yara_file_name}}</option>
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
</select>
|
</select>
|
||||||
|
@ -107,14 +111,15 @@
|
||||||
|
|
||||||
<hr>
|
<hr>
|
||||||
|
|
||||||
|
<h6>Custom YARA rules:</h6>
|
||||||
<div class="row" id="textarea">
|
<div class="row" id="textarea">
|
||||||
<textarea class="form-control mx-3" id="text_input" name="yara_custom_rule" placeholder="Enter your own YARA rule" rows="5"></textarea>
|
<textarea class="form-control mx-3" id="text_input" name="yara_custom_rule" placeholder="Enter your own YARA rule" rows="5">{%if dict_tracker%}{%if dict_tracker['type']=='yara' and dict_tracker['content']%}{{dict_tracker['content']}}{%endif%}{%endif%}</textarea>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<br>
|
<br>
|
||||||
<button class="btn btn-success mt-2">
|
<button class="btn btn-success mt-2">
|
||||||
<i class="fas fa-plus"></i> Add Tracker
|
<i class="fas fa-plus"></i> {%if dict_tracker%}Edit{%else%}Create{%endif%} Tracker
|
||||||
</button>
|
</button>
|
||||||
|
|
||||||
</form>
|
</form>
|
||||||
|
@ -139,7 +144,7 @@ $(document).ready(function(){
|
||||||
$("#page-Tracker").addClass("active");
|
$("#page-Tracker").addClass("active");
|
||||||
$("#nav_manual_crawler").addClass("active");
|
$("#nav_manual_crawler").addClass("active");
|
||||||
$("#tracker_desc").hide();
|
$("#tracker_desc").hide();
|
||||||
$("#term").hide();
|
$("#tracker").hide();
|
||||||
$("#nb_word").hide();
|
$("#nb_word").hide();
|
||||||
$("#yara_rule").hide();
|
$("#yara_rule").hide();
|
||||||
|
|
||||||
|
@ -148,30 +153,38 @@ $(document).ready(function(){
|
||||||
if (tracker_type=="word") {
|
if (tracker_type=="word") {
|
||||||
$("#tracker_desc").text("Token to track. You need to use a regex if you want to use one of the following special characters [<>~!?@#$%^&*|()_-+={}\":;,.\'\n\r\t]/\\ ");
|
$("#tracker_desc").text("Token to track. You need to use a regex if you want to use one of the following special characters [<>~!?@#$%^&*|()_-+={}\":;,.\'\n\r\t]/\\ ");
|
||||||
$("#tracker_desc").show();
|
$("#tracker_desc").show();
|
||||||
$("#term").show();
|
$("#tracker").show();
|
||||||
$("#nb_word").hide();
|
$("#nb_word").hide();
|
||||||
$("#yara_rule").hide();
|
$("#yara_rule").hide();
|
||||||
} else if (tracker_type=="set") {
|
} else if (tracker_type=="set") {
|
||||||
$("#tracker_desc").text("Set of Terms to track (space separated). This tracker is used to check if an item contain one or more terms specified in a set. If an item contain NB unique terms (by default NB of unique keywords = 1), this tracker is triggered. You need to use a regex if you want to use one of the following special characters [<>~!?@#$%^&*|()_-+={}\":;,.\'\n\r\t]/\\ ");
|
$("#tracker_desc").text("Set of Terms to track (space separated). This tracker is used to check if an item contain one or more terms specified in a set. If an item contain NB unique terms (by default NB of unique keywords = 1), this tracker is triggered. You need to use a regex if you want to use one of the following special characters [<>~!?@#$%^&*|()_-+={}\":;,.\'\n\r\t]/\\ ");
|
||||||
$("#tracker_desc").show();
|
$("#tracker_desc").show();
|
||||||
$("#term").show();
|
$("#tracker").show();
|
||||||
$("#nb_word").show();
|
$("#nb_word").show();
|
||||||
$("#yara_rule").hide();
|
$("#yara_rule").hide();
|
||||||
} else if (tracker_type=="regex") {
|
} else if (tracker_type=="regex") {
|
||||||
$("#tracker_desc").text("Enter a valid Python regex");
|
$("#tracker_desc").text("Enter a valid Python regex");
|
||||||
$("#tracker_desc").show();
|
$("#tracker_desc").show();
|
||||||
$("#term").show();
|
$("#tracker").show();
|
||||||
$("#nb_word").hide();
|
$("#nb_word").hide();
|
||||||
$("#yara_rule").hide();
|
$("#yara_rule").hide();
|
||||||
} else if (tracker_type=="yara") {
|
} else if (tracker_type=="yara") {
|
||||||
$("#tracker_desc").text("Select a default yara rule or create your own rule:");
|
$("#tracker_desc").text("Select a default yara rule or create your own rule:");
|
||||||
$("#tracker_desc").show();
|
$("#tracker_desc").show();
|
||||||
$("#term").hide();
|
$("#tracker").hide();
|
||||||
$("#nb_word").hide();
|
$("#nb_word").hide();
|
||||||
$("#yara_rule").show();
|
$("#yara_rule").show();
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
{%if dict_tracker%}
|
||||||
|
$('#tracker_type').val('{{dict_tracker['type']}}').change();
|
||||||
|
|
||||||
|
{%if dict_tracker['type']=='yara' and dict_tracker['yara_file']%}
|
||||||
|
$('#yara_default_rule').val('{{dict_tracker['yara_file']}}').change();
|
||||||
|
{%endif%}
|
||||||
|
{%endif%}
|
||||||
|
|
||||||
});
|
});
|
||||||
|
|
||||||
function toggle_sidebar(){
|
function toggle_sidebar(){
|
|
@ -171,10 +171,14 @@
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<a href="{{ url_for('hunter.delete_tracker') }}?uuid={{tracker_metadata['uuid']}}" class="float-right" style="font-size: 15px">
|
<div class="d-flex flex-row-reverse">
|
||||||
<button class='btn btn-danger'><i class="fas fa-trash-alt"></i>
|
<a href="{{ url_for('hunter.delete_tracker') }}?uuid={{tracker_metadata['uuid']}}" style="font-size: 15px">
|
||||||
</button>
|
<button class='btn btn-danger'><i class="fas fa-trash-alt"></i></button>
|
||||||
</a>
|
</a>
|
||||||
|
<a href="{{ url_for('hunter.edit_tracked_menu') }}?uuid={{tracker_metadata['uuid']}}" class="mx-2" style="font-size: 15px">
|
||||||
|
<button class='btn btn-info'>Edit Tracker <i class="fas fa-pencil-alt"></i></button>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
|
||||||
{%if yara_rule_content%}
|
{%if yara_rule_content%}
|
||||||
<p class="my-0"></br></br><pre class="border bg-light">{{ yara_rule_content }}</pre></p>
|
<p class="my-0"></br></br><pre class="border bg-light">{{ yara_rule_content }}</pre></p>
|
||||||
|
|
Loading…
Reference in New Issue