mirror of https://github.com/CIRCL/AIL-framework
commit
65eb084e41
|
@ -9,6 +9,8 @@ import redis
|
||||||
import uuid
|
import uuid
|
||||||
import yara
|
import yara
|
||||||
import datetime
|
import datetime
|
||||||
|
import base64
|
||||||
|
|
||||||
|
|
||||||
from flask import escape
|
from flask import escape
|
||||||
|
|
||||||
|
@ -159,6 +161,34 @@ def get_tracker_metadata(tracker_uuid, user_id=False, description=False, level=F
|
||||||
|
|
||||||
return dict_uuid
|
return dict_uuid
|
||||||
|
|
||||||
|
def get_tracker_metadata_api(request_dict):
|
||||||
|
tracker_uuid = request_dict.get('tracker_uuid', None)
|
||||||
|
if not request_dict:
|
||||||
|
return {'status': 'error', 'reason': 'Malformed JSON'}, 400
|
||||||
|
if not tracker_uuid:
|
||||||
|
return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400
|
||||||
|
if not is_valid_uuid_v4(tracker_uuid):
|
||||||
|
return {"status": "error", "reason": "Invalid Tracker UUID"}, 400
|
||||||
|
if not r_serv_tracker.exists(f'tracker:{tracker_uuid}'):
|
||||||
|
return {'status': 'error', 'reason': 'Tracker not found'}, 404
|
||||||
|
|
||||||
|
dict_tracker = {'status': 'success',
|
||||||
|
'uuid': tracker_uuid,
|
||||||
|
'user_id': get_tracker_user_id(tracker_uuid),
|
||||||
|
'tracker': get_tracker_by_uuid(tracker_uuid),
|
||||||
|
'type': get_tracker_type(tracker_uuid),
|
||||||
|
'date': get_tracker_date(tracker_uuid),
|
||||||
|
'first_seen': get_tracker_first_seen(tracker_uuid),
|
||||||
|
'last_seen': get_tracker_last_seen(tracker_uuid),
|
||||||
|
'level': get_tracker_level(tracker_uuid),
|
||||||
|
'mails': get_tracker_mails(tracker_uuid),
|
||||||
|
'tags': get_tracker_tags(tracker_uuid),
|
||||||
|
'description': get_tracker_description(tracker_uuid),
|
||||||
|
'webhook': get_tracker_webhook(tracker_uuid)
|
||||||
|
}
|
||||||
|
|
||||||
|
return dict_tracker, 200
|
||||||
|
|
||||||
# tracker sparkline
|
# tracker sparkline
|
||||||
def get_tracker_sparkline(tracker_uuid, num_day=6):
|
def get_tracker_sparkline(tracker_uuid, num_day=6):
|
||||||
date_range_sparkline = Date.get_date_range(num_day)
|
date_range_sparkline = Date.get_date_range(num_day)
|
||||||
|
@ -278,13 +308,13 @@ def is_tracker_in_user_level(tracker, tracker_type, user_id):
|
||||||
|
|
||||||
def api_is_allowed_to_edit_tracker(tracker_uuid, user_id):
|
def api_is_allowed_to_edit_tracker(tracker_uuid, user_id):
|
||||||
if not is_valid_uuid_v4(tracker_uuid):
|
if not is_valid_uuid_v4(tracker_uuid):
|
||||||
return ({"status": "error", "reason": "Invalid uuid"}, 400)
|
return {"status": "error", "reason": "Invalid uuid"}, 400
|
||||||
tracker_creator = r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'user_id')
|
tracker_creator = r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'user_id')
|
||||||
if not tracker_creator:
|
if not tracker_creator:
|
||||||
return ({"status": "error", "reason": "Unknown uuid"}, 404)
|
return {"status": "error", "reason": "Unknown uuid"}, 404
|
||||||
if not is_in_role(user_id, 'admin') and user_id != tracker_creator:
|
if not is_in_role(user_id, 'admin') and user_id != tracker_creator:
|
||||||
return ({"status": "error", "reason": "Access Denied"}, 403)
|
return {"status": "error", "reason": "Access Denied"}, 403
|
||||||
return ({"uuid": tracker_uuid}, 200)
|
return {"uuid": tracker_uuid}, 200
|
||||||
|
|
||||||
|
|
||||||
##-- ACL --##
|
##-- ACL --##
|
||||||
|
@ -335,14 +365,14 @@ def fix_all_tracker_uuid_list():
|
||||||
def api_validate_tracker_to_add(tracker , tracker_type, nb_words=1):
|
def api_validate_tracker_to_add(tracker , tracker_type, nb_words=1):
|
||||||
if tracker_type=='regex':
|
if tracker_type=='regex':
|
||||||
if not is_valid_regex(tracker):
|
if not is_valid_regex(tracker):
|
||||||
return ({"status": "error", "reason": "Invalid regex"}, 400)
|
return {"status": "error", "reason": "Invalid regex"}, 400
|
||||||
elif tracker_type=='word' or tracker_type=='set':
|
elif tracker_type=='word' or tracker_type=='set':
|
||||||
# force lowercase
|
# force lowercase
|
||||||
tracker = tracker.lower()
|
tracker = tracker.lower()
|
||||||
word_set = set(tracker)
|
word_set = set(tracker)
|
||||||
set_inter = word_set.intersection(special_characters)
|
set_inter = word_set.intersection(special_characters)
|
||||||
if set_inter:
|
if set_inter:
|
||||||
return ({"status": "error", "reason": f'special character(s) not allowed: {set_inter}', "message": "Please use a python regex or remove all special characters"}, 400)
|
return {"status": "error", "reason": f'special character(s) not allowed: {set_inter}', "message": "Please use a python regex or remove all special characters"}, 400
|
||||||
words = tracker.split()
|
words = tracker.split()
|
||||||
# not a word
|
# not a word
|
||||||
if tracker_type=='word' and len(words)>1:
|
if tracker_type=='word' and len(words)>1:
|
||||||
|
@ -368,13 +398,13 @@ def api_validate_tracker_to_add(tracker , tracker_type, nb_words=1):
|
||||||
|
|
||||||
elif tracker_type=='yara_custom':
|
elif tracker_type=='yara_custom':
|
||||||
if not is_valid_yara_rule(tracker):
|
if not is_valid_yara_rule(tracker):
|
||||||
return ({"status": "error", "reason": "Invalid custom Yara Rule"}, 400)
|
return {"status": "error", "reason": "Invalid custom Yara Rule"}, 400
|
||||||
elif tracker_type=='yara_default':
|
elif tracker_type=='yara_default':
|
||||||
if not is_valid_default_yara_rule(tracker):
|
if not is_valid_default_yara_rule(tracker):
|
||||||
return ({"status": "error", "reason": "The Yara Rule doesn't exist"}, 400)
|
return {"status": "error", "reason": "The Yara Rule doesn't exist"}, 400
|
||||||
else:
|
else:
|
||||||
return ({"status": "error", "reason": "Incorrect type"}, 400)
|
return {"status": "error", "reason": "Incorrect type"}, 400
|
||||||
return ({"status": "success", "tracker": tracker, "type": tracker_type}, 200)
|
return {"status": "success", "tracker": tracker, "type": tracker_type}, 200
|
||||||
|
|
||||||
def create_tracker(tracker, tracker_type, user_id, level, tags, mails, description, webhook, dashboard=0, tracker_uuid=None, sources=[]):
|
def create_tracker(tracker, tracker_type, user_id, level, tags, mails, description, webhook, dashboard=0, tracker_uuid=None, sources=[]):
|
||||||
# edit tracker
|
# edit tracker
|
||||||
|
@ -683,17 +713,36 @@ def api_get_default_rule_content(default_yara_rule):
|
||||||
yara_dir = get_yara_rules_default_dir()
|
yara_dir = get_yara_rules_default_dir()
|
||||||
filename = os.path.join(yara_dir, default_yara_rule)
|
filename = os.path.join(yara_dir, default_yara_rule)
|
||||||
filename = os.path.realpath(filename)
|
filename = os.path.realpath(filename)
|
||||||
|
|
||||||
# incorrect filename
|
|
||||||
if not os.path.commonprefix([filename, yara_dir]) == yara_dir:
|
if not os.path.commonprefix([filename, yara_dir]) == yara_dir:
|
||||||
return ({'status': 'error', 'reason': 'file transversal detected'}, 400)
|
return {'status': 'error', 'reason': 'file traversal detected'}, 400
|
||||||
|
|
||||||
if not os.path.isfile(filename):
|
if not os.path.isfile(filename):
|
||||||
return ({'status': 'error', 'reason': 'yara rule not found'}, 400)
|
return {'status': 'error', 'reason': 'yara rule not found'}, 400
|
||||||
|
|
||||||
with open(filename, 'r') as f:
|
with open(filename, 'r') as f:
|
||||||
rule_content = f.read()
|
rule_content = f.read()
|
||||||
return ({'rule_name': default_yara_rule, 'content': rule_content}, 200)
|
return {'rule_name': default_yara_rule, 'content': rule_content}, 200
|
||||||
|
|
||||||
|
|
||||||
|
def get_yara_rule_content_restapi(request_dict):
|
||||||
|
rule_name = request_dict.get('rule_name', None)
|
||||||
|
if not request_dict:
|
||||||
|
return {'status': 'error', 'reason': 'Malformed JSON'}, 400
|
||||||
|
if not rule_name:
|
||||||
|
return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400
|
||||||
|
yara_dir = get_yara_rules_dir()
|
||||||
|
filename = os.path.join(yara_dir, rule_name)
|
||||||
|
filename = os.path.realpath(filename)
|
||||||
|
if not os.path.commonprefix([filename, yara_dir]) == yara_dir:
|
||||||
|
return {'status': 'error', 'reason': 'File Path Traversal'}, 400
|
||||||
|
if not os.path.isfile(filename):
|
||||||
|
return {'status': 'error', 'reason': 'yara rule not found'}, 400
|
||||||
|
with open(filename, 'r') as f:
|
||||||
|
rule_content = f.read()
|
||||||
|
rule_content = base64.b64encode((rule_content.encode('utf-8'))).decode('UTF-8')
|
||||||
|
return {'status': 'success', 'content': rule_content}, 200
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
##-- YARA --##
|
##-- YARA --##
|
||||||
|
|
||||||
|
@ -1086,9 +1135,9 @@ def get_retro_hunt_nb_item_by_day(l_task_uuid, date_from=None, date_to=None):
|
||||||
## API ##
|
## API ##
|
||||||
def api_check_retro_hunt_task_uuid(task_uuid):
|
def api_check_retro_hunt_task_uuid(task_uuid):
|
||||||
if not is_valid_uuid_v4(task_uuid):
|
if not is_valid_uuid_v4(task_uuid):
|
||||||
return ({"status": "error", "reason": "Invalid uuid"}, 400)
|
return {"status": "error", "reason": "Invalid uuid"}, 400
|
||||||
if not r_serv_tracker.exists(f'tracker:retro_hunt:task:{task_uuid}'):
|
if not r_serv_tracker.exists(f'tracker:retro_hunt:task:{task_uuid}'):
|
||||||
return ({"status": "error", "reason": "Unknown uuid"}, 404)
|
return {"status": "error", "reason": "Unknown uuid"}, 404
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def api_get_retro_hunt_items(dict_input):
|
def api_get_retro_hunt_items(dict_input):
|
||||||
|
@ -1116,7 +1165,7 @@ def api_get_retro_hunt_items(dict_input):
|
||||||
res_dict['date_from'] = date_from
|
res_dict['date_from'] = date_from
|
||||||
res_dict['date_to'] = date_to
|
res_dict['date_to'] = date_to
|
||||||
res_dict['items'] = all_items_id
|
res_dict['items'] = all_items_id
|
||||||
return (res_dict, 200)
|
return res_dict, 200
|
||||||
|
|
||||||
def api_pause_retro_hunt_task(task_uuid):
|
def api_pause_retro_hunt_task(task_uuid):
|
||||||
res = api_check_retro_hunt_task_uuid(task_uuid)
|
res = api_check_retro_hunt_task_uuid(task_uuid)
|
||||||
|
@ -1124,9 +1173,9 @@ def api_pause_retro_hunt_task(task_uuid):
|
||||||
return res
|
return res
|
||||||
task_state = get_retro_hunt_task_state(task_uuid)
|
task_state = get_retro_hunt_task_state(task_uuid)
|
||||||
if task_state not in ['pending', 'running']:
|
if task_state not in ['pending', 'running']:
|
||||||
return ({"status": "error", "reason": f"Task {task_uuid} not paused, current state: {task_state}"}, 400)
|
return {"status": "error", "reason": f"Task {task_uuid} not paused, current state: {task_state}"}, 400
|
||||||
pause_retro_hunt_task(task_uuid)
|
pause_retro_hunt_task(task_uuid)
|
||||||
return (task_uuid, 200)
|
return task_uuid, 200
|
||||||
|
|
||||||
def api_resume_retro_hunt_task(task_uuid):
|
def api_resume_retro_hunt_task(task_uuid):
|
||||||
res = api_check_retro_hunt_task_uuid(task_uuid)
|
res = api_check_retro_hunt_task_uuid(task_uuid)
|
||||||
|
@ -1134,9 +1183,9 @@ def api_resume_retro_hunt_task(task_uuid):
|
||||||
return res
|
return res
|
||||||
task_state = get_retro_hunt_task_state(task_uuid)
|
task_state = get_retro_hunt_task_state(task_uuid)
|
||||||
if not r_serv_tracker.sismember('tracker:retro_hunt:task:paused', task_uuid):
|
if not r_serv_tracker.sismember('tracker:retro_hunt:task:paused', task_uuid):
|
||||||
return ({"status": "error", "reason": f"Task {task_uuid} not paused, current state: {get_retro_hunt_task_state(task_uuid)}"}, 400)
|
return {"status": "error", "reason": f"Task {task_uuid} not paused, current state: {get_retro_hunt_task_state(task_uuid)}"}, 400
|
||||||
resume_retro_hunt_task(task_uuid)
|
resume_retro_hunt_task(task_uuid)
|
||||||
return (task_uuid, 200)
|
return task_uuid, 200
|
||||||
|
|
||||||
def api_validate_rule_to_add(rule, rule_type):
|
def api_validate_rule_to_add(rule, rule_type):
|
||||||
if rule_type=='yara_custom':
|
if rule_type=='yara_custom':
|
||||||
|
|
|
@ -221,7 +221,7 @@ def verify_sources_list(sources):
|
||||||
all_sources = get_all_items_sources()
|
all_sources = get_all_items_sources()
|
||||||
for source in sources:
|
for source in sources:
|
||||||
if source not in all_sources:
|
if source not in all_sources:
|
||||||
return ({'status': 'error', 'reason': 'Invalid source', 'value': source}, 400)
|
return {'status': 'error', 'reason': 'Invalid source', 'value': source}, 400
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def get_all_items_metadata_dict(list_id):
|
def get_all_items_metadata_dict(list_id):
|
||||||
|
|
|
@ -60,6 +60,9 @@ def get_item_date(item_id, add_separator=False):
|
||||||
def get_source(item_id):
|
def get_source(item_id):
|
||||||
return item_basic.get_source(item_id)
|
return item_basic.get_source(item_id)
|
||||||
|
|
||||||
|
def get_all_sources():
|
||||||
|
return item_basic.get_all_items_sources(r_list=True)
|
||||||
|
|
||||||
def get_item_basename(item_id):
|
def get_item_basename(item_id):
|
||||||
return os.path.basename(item_id)
|
return os.path.basename(item_id)
|
||||||
|
|
||||||
|
@ -88,14 +91,12 @@ def get_item_metadata(item_id, item_content=None):
|
||||||
# encoding
|
# encoding
|
||||||
# language
|
# language
|
||||||
# lines info
|
# lines info
|
||||||
|
item_metadata = {'date': get_item_date(item_id, add_separator=True),
|
||||||
item_metadata = {}
|
'source': get_source(item_id),
|
||||||
item_metadata['date'] = get_item_date(item_id, add_separator=True)
|
'size': get_item_size(item_id),
|
||||||
item_metadata['source'] = get_source(item_id)
|
'encoding': get_item_encoding(item_id),
|
||||||
item_metadata['size'] = get_item_size(item_id)
|
'lines': get_lines_info(item_id, item_content=item_content)
|
||||||
item_metadata['encoding'] = get_item_encoding(item_id)
|
}
|
||||||
item_metadata['lines'] = get_lines_info(item_id, item_content=item_content)
|
|
||||||
|
|
||||||
return item_metadata
|
return item_metadata
|
||||||
|
|
||||||
def get_item_parent(item_id):
|
def get_item_parent(item_id):
|
||||||
|
@ -166,13 +167,13 @@ def get_item_languages(item_id, min_len=600, num_langs=3, min_proportion=0.2, mi
|
||||||
# API
|
# API
|
||||||
def get_item(request_dict):
|
def get_item(request_dict):
|
||||||
if not request_dict:
|
if not request_dict:
|
||||||
return Response({'status': 'error', 'reason': 'Malformed JSON'}, 400)
|
return {'status': 'error', 'reason': 'Malformed JSON'}, 400
|
||||||
|
|
||||||
item_id = request_dict.get('id', None)
|
item_id = request_dict.get('id', None)
|
||||||
if not item_id:
|
if not item_id:
|
||||||
return ( {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 )
|
return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400
|
||||||
if not exist_item(item_id):
|
if not exist_item(item_id):
|
||||||
return ( {'status': 'error', 'reason': 'Item not found'}, 404 )
|
return {'status': 'error', 'reason': 'Item not found'}, 404
|
||||||
|
|
||||||
dict_item = {}
|
dict_item = {}
|
||||||
dict_item['id'] = item_id
|
dict_item['id'] = item_id
|
||||||
|
@ -217,9 +218,40 @@ def get_item(request_dict):
|
||||||
if request_dict['cryptocurrency'].get('bitcoin'):
|
if request_dict['cryptocurrency'].get('bitcoin'):
|
||||||
dict_item['cryptocurrency']['bitcoin'] = get_item_bitcoin(item_id)
|
dict_item['cryptocurrency']['bitcoin'] = get_item_bitcoin(item_id)
|
||||||
|
|
||||||
return (dict_item, 200)
|
return dict_item, 200
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def get_item_content_encoded_text(request_dict):
|
||||||
|
item_id = request_dict.get('id', None)
|
||||||
|
if not request_dict:
|
||||||
|
return {'status': 'error', 'reason': 'Malformed JSON'}, 400
|
||||||
|
if not item_id:
|
||||||
|
return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400
|
||||||
|
if not exist_item(item_id):
|
||||||
|
return {'status': 'error', 'reason': 'Item not found'}, 404
|
||||||
|
|
||||||
|
item_content = get_item_content(item_id)
|
||||||
|
item_content = base64.b64encode((item_content.encode('utf-8'))).decode('UTF-8')
|
||||||
|
return {'status': 'success', 'content': item_content}, 200
|
||||||
|
|
||||||
|
|
||||||
|
def get_item_sources():
|
||||||
|
item_content = {'sources': get_all_sources()}
|
||||||
|
return item_content, 200
|
||||||
|
|
||||||
|
def check_item_source(request_dict):
|
||||||
|
source = request_dict.get('source', None)
|
||||||
|
if not request_dict:
|
||||||
|
return {'status': 'error', 'reason': 'Malformed JSON'}, 400
|
||||||
|
if not source:
|
||||||
|
return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400
|
||||||
|
|
||||||
|
all_sources = item_basic.get_all_items_sources()
|
||||||
|
|
||||||
|
if source not in all_sources:
|
||||||
|
return {'status': 'error', 'reason': 'Invalid source', 'provide': source}, 400
|
||||||
|
return {'status': 'success', 'reason': 'Valid source', 'provide': source}, 200
|
||||||
###
|
###
|
||||||
### correlation
|
### correlation
|
||||||
###
|
###
|
||||||
|
|
|
@ -57,14 +57,14 @@ def is_in_role(user_id, role):
|
||||||
|
|
||||||
def check_term_uuid_valid_access(term_uuid, user_id):
|
def check_term_uuid_valid_access(term_uuid, user_id):
|
||||||
if not is_valid_uuid_v4(term_uuid):
|
if not is_valid_uuid_v4(term_uuid):
|
||||||
return ({"status": "error", "reason": "Invalid uuid"}, 400)
|
return {"status": "error", "reason": "Invalid uuid"}, 400
|
||||||
level = r_serv_term.hget('tracker:{}'.format(term_uuid), 'level')
|
level = r_serv_term.hget('tracker:{}'.format(term_uuid), 'level')
|
||||||
if not level:
|
if not level:
|
||||||
return ({"status": "error", "reason": "Unknown uuid"}, 404)
|
return {"status": "error", "reason": "Unknown uuid"}, 404
|
||||||
if level == 0:
|
if level == 0:
|
||||||
if r_serv_term.hget('tracker:{}'.format(term_uuid), 'user_id') != user_id:
|
if r_serv_term.hget('tracker:{}'.format(term_uuid), 'user_id') != user_id:
|
||||||
if not is_in_role(user_id, 'admin'):
|
if not is_in_role(user_id, 'admin'):
|
||||||
return ({"status": "error", "reason": "Unknown uuid"}, 404)
|
return {"status": "error", "reason": "Unknown uuid"}, 404
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
@ -78,7 +78,7 @@ def is_valid_mail(email):
|
||||||
def verify_mail_list(mail_list):
|
def verify_mail_list(mail_list):
|
||||||
for mail in mail_list:
|
for mail in mail_list:
|
||||||
if not is_valid_mail(mail):
|
if not is_valid_mail(mail):
|
||||||
return ({'status': 'error', 'reason': 'Invalid email', 'value': mail}, 400)
|
return {'status': 'error', 'reason': 'Invalid email', 'value': mail}, 400
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def is_valid_regex(term_regex):
|
def is_valid_regex(term_regex):
|
||||||
|
@ -148,14 +148,19 @@ def is_term_tracked_in_user_level(term, term_type, user_id):
|
||||||
def parse_json_term_to_add(dict_input, user_id):
|
def parse_json_term_to_add(dict_input, user_id):
|
||||||
term = dict_input.get('term', None)
|
term = dict_input.get('term', None)
|
||||||
if not term:
|
if not term:
|
||||||
return ({"status": "error", "reason": "Term not provided"}, 400)
|
return {"status": "error", "reason": "Term not provided"}, 400
|
||||||
term_type = dict_input.get('type', None)
|
term_type = dict_input.get('type', None)
|
||||||
if not term_type:
|
if not term_type:
|
||||||
return ({"status": "error", "reason": "Term type not provided"}, 400)
|
return {"status": "error", "reason": "Term type not provided"}, 400
|
||||||
|
|
||||||
nb_words = dict_input.get('nb_words', 1)
|
nb_words = dict_input.get('nb_words', 1)
|
||||||
|
|
||||||
description = dict_input.get('description', '')
|
description = dict_input.get('description', '')
|
||||||
description = escape(description)
|
description = escape(description)
|
||||||
|
|
||||||
|
webhook = dict_input.get('webhook', '')
|
||||||
|
webhook = escape(webhook)
|
||||||
|
|
||||||
res = parse_tracked_term_to_add(term , term_type, nb_words=nb_words)
|
res = parse_tracked_term_to_add(term , term_type, nb_words=nb_words)
|
||||||
if res[1]!=200:
|
if res[1]!=200:
|
||||||
return res
|
return res
|
||||||
|
@ -168,7 +173,6 @@ def parse_json_term_to_add(dict_input, user_id):
|
||||||
if res:
|
if res:
|
||||||
return res
|
return res
|
||||||
|
|
||||||
## TODO: add dashboard key
|
|
||||||
level = dict_input.get('level', 1)
|
level = dict_input.get('level', 1)
|
||||||
try:
|
try:
|
||||||
level = int(level)
|
level = int(level)
|
||||||
|
@ -180,27 +184,27 @@ def parse_json_term_to_add(dict_input, user_id):
|
||||||
# check if term already tracked in global
|
# check if term already tracked in global
|
||||||
if level==1:
|
if level==1:
|
||||||
if is_term_tracked_in_global_level(term, term_type):
|
if is_term_tracked_in_global_level(term, term_type):
|
||||||
return ({"status": "error", "reason": "Term already tracked"}, 409)
|
return {"status": "error", "reason": "Term already tracked"}, 409
|
||||||
else:
|
else:
|
||||||
if is_term_tracked_in_user_level(term, term_type, user_id):
|
if is_term_tracked_in_user_level(term, term_type, user_id):
|
||||||
return ({"status": "error", "reason": "Term already tracked"}, 409)
|
return {"status": "error", "reason": "Term already tracked"}, 409
|
||||||
|
|
||||||
term_uuid = add_tracked_term(term , term_type, user_id, level, tags, mails, description)
|
term_uuid = add_tracked_term(term , term_type, user_id, level, tags, mails, description,webhook)
|
||||||
|
|
||||||
return ({'term': term, 'type': term_type, 'uuid': term_uuid}, 200)
|
return {'term': term, 'type': term_type, 'uuid': term_uuid}, 200
|
||||||
|
|
||||||
|
|
||||||
def parse_tracked_term_to_add(term , term_type, nb_words=1):
|
def parse_tracked_term_to_add(term , term_type, nb_words=1):
|
||||||
if term_type=='regex':
|
if term_type=='regex':
|
||||||
if not is_valid_regex(term):
|
if not is_valid_regex(term):
|
||||||
return ({"status": "error", "reason": "Invalid regex"}, 400)
|
return {"status": "error", "reason": "Invalid regex"}, 400
|
||||||
elif term_type=='word' or term_type=='set':
|
elif term_type=='word' or term_type=='set':
|
||||||
# force lowercase
|
# force lowercase
|
||||||
term = term.lower()
|
term = term.lower()
|
||||||
word_set = set(term)
|
word_set = set(term)
|
||||||
set_inter = word_set.intersection(special_characters)
|
set_inter = word_set.intersection(special_characters)
|
||||||
if set_inter:
|
if set_inter:
|
||||||
return ({"status": "error", "reason": "special character not allowed", "message": "Please use a regex or remove all special characters"}, 400)
|
return {"status": "error", "reason": "special character not allowed", "message": "Please use a regex or remove all special characters"}, 400
|
||||||
words = term.split()
|
words = term.split()
|
||||||
# not a word
|
# not a word
|
||||||
if term_type=='word' and len(words)>1:
|
if term_type=='word' and len(words)>1:
|
||||||
|
@ -226,15 +230,15 @@ def parse_tracked_term_to_add(term , term_type, nb_words=1):
|
||||||
|
|
||||||
elif term_type=='yara_custom':
|
elif term_type=='yara_custom':
|
||||||
if not Tracker.is_valid_yara_rule(term):
|
if not Tracker.is_valid_yara_rule(term):
|
||||||
return ({"status": "error", "reason": "Invalid custom Yara Rule"}, 400)
|
return {"status": "error", "reason": "Invalid custom Yara Rule"}, 400
|
||||||
elif term_type=='yara_default':
|
elif term_type=='yara_default':
|
||||||
if not Tracker.is_valid_default_yara_rule(term):
|
if not Tracker.is_valid_default_yara_rule(term):
|
||||||
return ({"status": "error", "reason": "The Yara Rule doesn't exist"}, 400)
|
return {"status": "error", "reason": "The Yara Rule doesn't exist"}, 400
|
||||||
else:
|
else:
|
||||||
return ({"status": "error", "reason": "Incorrect type"}, 400)
|
return {"status": "error", "reason": "Incorrect type"}, 400
|
||||||
return ({"status": "success", "term": term, "type": term_type}, 200)
|
return {"status": "success", "term": term, "type": term_type}, 200
|
||||||
|
|
||||||
def add_tracked_term(term , term_type, user_id, level, tags, mails, description, dashboard=0):
|
def add_tracked_term(term , term_type, user_id, level, tags, mails, description,webhook, dashboard=0):
|
||||||
|
|
||||||
term_uuid = str(uuid.uuid4())
|
term_uuid = str(uuid.uuid4())
|
||||||
|
|
||||||
|
@ -254,6 +258,9 @@ def add_tracked_term(term , term_type, user_id, level, tags, mails, description,
|
||||||
if description:
|
if description:
|
||||||
r_serv_term.hset('tracker:{}'.format(term_uuid), 'description', description)
|
r_serv_term.hset('tracker:{}'.format(term_uuid), 'description', description)
|
||||||
|
|
||||||
|
if webhook:
|
||||||
|
r_serv_term.hset('tracker:{}'.format(term_uuid), 'webhook', webhook)
|
||||||
|
|
||||||
# create all term set
|
# create all term set
|
||||||
r_serv_term.sadd('all:tracker:{}'.format(term_type), term)
|
r_serv_term.sadd('all:tracker:{}'.format(term_type), term)
|
||||||
|
|
||||||
|
@ -281,14 +288,15 @@ def add_tracked_term(term , term_type, user_id, level, tags, mails, description,
|
||||||
|
|
||||||
return term_uuid
|
return term_uuid
|
||||||
|
|
||||||
|
|
||||||
def parse_tracked_term_to_delete(dict_input, user_id):
|
def parse_tracked_term_to_delete(dict_input, user_id):
|
||||||
term_uuid = dict_input.get("uuid", None)
|
term_uuid = dict_input.get("uuid", None)
|
||||||
res = check_term_uuid_valid_access(term_uuid, user_id)
|
res = check_term_uuid_valid_access(term_uuid, user_id)
|
||||||
if res:
|
if res:
|
||||||
return res
|
return res
|
||||||
|
|
||||||
delete_term(term_uuid)
|
delete_term(term_uuid)
|
||||||
return ({"uuid": term_uuid}, 200)
|
return {"uuid": term_uuid}, 200
|
||||||
|
|
||||||
|
|
||||||
# # TODO: MOVE IN TRACKER
|
# # TODO: MOVE IN TRACKER
|
||||||
def delete_term(term_uuid):
|
def delete_term(term_uuid):
|
||||||
|
@ -328,7 +336,6 @@ def delete_term(term_uuid):
|
||||||
r_serv_term.delete('tracker:sources:{}'.format(term_uuid))
|
r_serv_term.delete('tracker:sources:{}'.format(term_uuid))
|
||||||
|
|
||||||
# remove item set
|
# remove item set
|
||||||
#########################3
|
|
||||||
all_item_date = r_serv_term.zrange(f'tracker:stat:{term_uuid}', 0, -1, withscores=True)
|
all_item_date = r_serv_term.zrange(f'tracker:stat:{term_uuid}', 0, -1, withscores=True)
|
||||||
if all_item_date:
|
if all_item_date:
|
||||||
all_item_date = dict(all_item_date)
|
all_item_date = dict(all_item_date)
|
||||||
|
@ -426,12 +433,8 @@ def parse_get_tracker_term_item(dict_input, user_id):
|
||||||
all_item_id = Tracker.get_tracker_items_by_daterange(term_uuid, date_from, date_to)
|
all_item_id = Tracker.get_tracker_items_by_daterange(term_uuid, date_from, date_to)
|
||||||
all_item_id = Item.get_item_list_desc(all_item_id)
|
all_item_id = Item.get_item_list_desc(all_item_id)
|
||||||
|
|
||||||
res_dict = {}
|
res_dict = {'uuid': term_uuid, 'date_from': date_from, 'date_to': date_to, 'items': all_item_id}
|
||||||
res_dict['uuid'] = term_uuid
|
return res_dict, 200
|
||||||
res_dict['date_from'] = date_from
|
|
||||||
res_dict['date_to'] = date_to
|
|
||||||
res_dict['items'] = all_item_id
|
|
||||||
return (res_dict, 200)
|
|
||||||
|
|
||||||
def get_tracked_term_first_seen(term_uuid):
|
def get_tracked_term_first_seen(term_uuid):
|
||||||
return Tracker.get_tracker_first_seen(term_uuid)
|
return Tracker.get_tracker_first_seen(term_uuid)
|
||||||
|
@ -444,7 +447,6 @@ def get_term_metedata(term_uuid, user_id=False, description=False, level=False,
|
||||||
dict_uuid['term'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'tracked')
|
dict_uuid['term'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'tracked')
|
||||||
dict_uuid['type'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'type')
|
dict_uuid['type'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'type')
|
||||||
dict_uuid['date'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'date')
|
dict_uuid['date'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'date')
|
||||||
dict_uuid['description'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'description')
|
|
||||||
dict_uuid['first_seen'] = get_tracked_term_first_seen(term_uuid)
|
dict_uuid['first_seen'] = get_tracked_term_first_seen(term_uuid)
|
||||||
dict_uuid['last_seen'] = get_tracked_term_last_seen(term_uuid)
|
dict_uuid['last_seen'] = get_tracked_term_last_seen(term_uuid)
|
||||||
if user_id:
|
if user_id:
|
||||||
|
@ -457,6 +459,8 @@ def get_term_metedata(term_uuid, user_id=False, description=False, level=False,
|
||||||
dict_uuid['tags'] = get_list_trackeed_term_tags(term_uuid)
|
dict_uuid['tags'] = get_list_trackeed_term_tags(term_uuid)
|
||||||
if sparkline:
|
if sparkline:
|
||||||
dict_uuid['sparkline'] = get_tracked_term_sparkline(term_uuid)
|
dict_uuid['sparkline'] = get_tracked_term_sparkline(term_uuid)
|
||||||
|
if description:
|
||||||
|
dict_uuid['description'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'description')
|
||||||
dict_uuid['uuid'] = term_uuid
|
dict_uuid['uuid'] = term_uuid
|
||||||
return dict_uuid
|
return dict_uuid
|
||||||
|
|
||||||
|
|
|
@ -112,11 +112,9 @@ def add_tracked_menu():
|
||||||
else:
|
else:
|
||||||
tracker = yara_default_rule
|
tracker = yara_default_rule
|
||||||
tracker_type='yara_default'
|
tracker_type='yara_default'
|
||||||
# #
|
|
||||||
|
|
||||||
if level == 'on':
|
if level == 'on':
|
||||||
level = 1
|
level = 1
|
||||||
|
|
||||||
if mails:
|
if mails:
|
||||||
mails = mails.split()
|
mails = mails.split()
|
||||||
if tags:
|
if tags:
|
||||||
|
|
|
@ -23,6 +23,7 @@ import Item
|
||||||
import Paste
|
import Paste
|
||||||
import Tag
|
import Tag
|
||||||
import Term
|
import Term
|
||||||
|
import Tracker
|
||||||
|
|
||||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'import'))
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'import'))
|
||||||
import importer
|
import importer
|
||||||
|
@ -305,7 +306,6 @@ def delete_item_tags():
|
||||||
@restApi.route("api/v1/get/item/content", methods=['POST'])
|
@restApi.route("api/v1/get/item/content", methods=['POST'])
|
||||||
@token_required('read_only')
|
@token_required('read_only')
|
||||||
def get_item_content():
|
def get_item_content():
|
||||||
|
|
||||||
data = request.get_json()
|
data = request.get_json()
|
||||||
item_id = data.get('id', None)
|
item_id = data.get('id', None)
|
||||||
req_data = {'id': item_id, 'date': False, 'content': True, 'tags': False}
|
req_data = {'id': item_id, 'date': False, 'content': True, 'tags': False}
|
||||||
|
@ -313,6 +313,32 @@ def get_item_content():
|
||||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
|
||||||
|
|
||||||
|
@restApi.route("api/v1/get/item/content/encoded/text", methods=['POST'])
|
||||||
|
@token_required('read_only')
|
||||||
|
def get_item_content_encoded_text():
|
||||||
|
data = request.get_json()
|
||||||
|
item_id = data.get('id', None)
|
||||||
|
req_data = {'id': item_id}
|
||||||
|
res = Item.get_item_content_encoded_text(req_data)
|
||||||
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
|
||||||
|
|
||||||
|
@restApi.route("api/v1/get/item/sources", methods=['GET'])
|
||||||
|
@token_required('read_only')
|
||||||
|
def get_item_sources():
|
||||||
|
res = Item.get_item_sources()
|
||||||
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@restApi.route("api/v1/get/item/source/check", methods=['POST'])
|
||||||
|
@token_required('read_only')
|
||||||
|
def get_check_item_source():
|
||||||
|
data = request.get_json()
|
||||||
|
source = data.get('source', None)
|
||||||
|
req_data = {'source': source}
|
||||||
|
res = Item.check_item_source(req_data)
|
||||||
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||||
# # # # # # # # # # # # # # TAGS # # # # # # # # # # # # # # # # #
|
# # # # # # # # # # # # # # TAGS # # # # # # # # # # # # # # # # #
|
||||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||||
|
@ -342,7 +368,7 @@ def add_tracker_term():
|
||||||
data = request.get_json()
|
data = request.get_json()
|
||||||
user_token = get_auth_from_header()
|
user_token = get_auth_from_header()
|
||||||
user_id = get_user_from_token(user_token)
|
user_id = get_user_from_token(user_token)
|
||||||
res = Term.parse_json_term_to_add(data, user_id)
|
res = Tracker.api_add_tracker(data, user_id)
|
||||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
|
||||||
@restApi.route("api/v1/delete/tracker", methods=['DELETE'])
|
@restApi.route("api/v1/delete/tracker", methods=['DELETE'])
|
||||||
|
@ -364,6 +390,24 @@ def get_tracker_term_item():
|
||||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
|
||||||
|
|
||||||
|
@restApi.route("api/v1/get/tracker/yara/content", methods=['POST'])
|
||||||
|
@token_required('read_only')
|
||||||
|
def get_default_yara_rule_content():
|
||||||
|
data = request.get_json()
|
||||||
|
rule_name = data.get('rule_name', None)
|
||||||
|
req_data = {'rule_name': rule_name}
|
||||||
|
res = Tracker.get_yara_rule_content_restapi(req_data)
|
||||||
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
|
||||||
|
|
||||||
|
@restApi.route("api/v1/get/tracker/metadata", methods=['POST'])
|
||||||
|
@token_required('read_only')
|
||||||
|
def get_tracker_metadata_api():
|
||||||
|
data = request.get_json()
|
||||||
|
tracker_uuid = data.get('tracker_uuid', None)
|
||||||
|
req_data = {'tracker_uuid': tracker_uuid}
|
||||||
|
res = Tracker.get_tracker_metadata_api(req_data)
|
||||||
|
return Response(json.dumps(res[0], indent=2, sort_keys=False), mimetype='application/json'), res[1]
|
||||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||||
# # # # # # # # # # # # CRYPTOCURRENCY # # # # # # # # # # # # # #
|
# # # # # # # # # # # # CRYPTOCURRENCY # # # # # # # # # # # # # #
|
||||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||||
|
|
Loading…
Reference in New Issue