CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

fix: [Tracker migration] clean old Term library

pull/594/head
Terrtia 2022-07-11 11:37:16 +02:00
parent 41a4278371
commit 8672671e51
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
5 changed files with 92 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
bin/DB_KVROCKS_MIGRATION.py
View File

@ -220,8 +220,11 @@ def tags_migration():
pass pass
def items_migration():
pass
def crawler_migration(): def crawler_migration():
pass pass
def domain_migration(): def domain_migration():
@ -244,7 +247,7 @@ if __name__ == '__main__':
core_migration() core_migration()
user_migration() user_migration()
#ail_2_ail_migration() #ail_2_ail_migration()
#trackers_migration() trackers_migration()
#investigations_migration() #investigations_migration()

34
bin/lib/Tracker.py
View File

@ -92,6 +92,18 @@ def get_all_tracker_by_type(tracker_type):
def get_all_tracker_by_type(tracker_type): def get_all_tracker_by_type(tracker_type):
return r_serv_tracker.smembers(f'all:tracker:{tracker_type}') return r_serv_tracker.smembers(f'all:tracker:{tracker_type}')
def get_user_trackers_uuid(user_id, tracker_type=None):
if tracker_type:
return r_serv_tracker.smembers(f'user:tracker:{user_id}:{tracker_type}')
else:
return r_serv_tracker.smembers(f'user:tracker:{user_id}')
def get_global_trackers_uuid(tracker_type=None):
if tracker_type:
return r_serv_tracker.smembers(f'global:tracker:{tracker_type}')
else:
return r_serv_tracker.smembers('global:tracker')
def get_tracker_by_uuid(tracker_uuid): def get_tracker_by_uuid(tracker_uuid):
return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'tracked') return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'tracked')
@ -169,6 +181,18 @@ def get_tracker_metadata(tracker_uuid, user_id=False, description=False, level=F
return dict_uuid return dict_uuid
def get_user_trackers_metadata(user_id, tracker_type=None):
meta_trackers = []
for tracker_uuid in get_user_trackers_uuid(user_id, tracker_type=None):
meta_trackers.append(get_tracker_metadata(tracker_uuid, tags=True, mails=True, sparkline=True))
return meta_trackers
def get_global_trackers_metadata(tracker_type=None):
meta_trackers = []
for tracker_uuid in get_global_trackers_uuid(tracker_type=None):
meta_trackers.append(get_tracker_metadata(tracker_uuid, tags=True, mails=True, sparkline=True))
return meta_trackers
def get_tracker_metadata_api(request_dict): def get_tracker_metadata_api(request_dict):
tracker_uuid = request_dict.get('tracker_uuid', None) tracker_uuid = request_dict.get('tracker_uuid', None)
if not request_dict: if not request_dict:
@ -326,6 +350,14 @@ def is_tracker_in_user_level(tracker, tracker_type, user_id):
return True return True
return False return False
## API ##
def api_check_tracker_uuid(tracker_uuid):
if not is_valid_uuid_v4(task_uuid):
return {"status": "error", "reason": "Invalid uuid"}, 400
if not r_serv_tracker.exists(f'tracker:{tracker_uuid}'):
return {"status": "error", "reason": "Unknown uuid"}, 404
return None
def api_is_allowed_to_edit_tracker(tracker_uuid, user_id): def api_is_allowed_to_edit_tracker(tracker_uuid, user_id):
if not is_valid_uuid_v4(tracker_uuid): if not is_valid_uuid_v4(tracker_uuid):
return {"status": "error", "reason": "Invalid uuid"}, 400 return {"status": "error", "reason": "Invalid uuid"}, 400
@ -1135,7 +1167,7 @@ def save_retro_hunt_match(task_uuid, id, object_type='item'):
if res == 1: if res == 1:
r_serv_tracker.zincrby(f'tracker:retro_hunt:task:stat:{task_uuid}', int(item_date), 1) r_serv_tracker.zincrby(f'tracker:retro_hunt:task:stat:{task_uuid}', int(item_date), 1)
# Add map obj_id -> task_uuid # Add map obj_id -> task_uuid
r_serv_tracker.sadd(f'obj:retro_hunt:item:{item_id}', task_uuid) r_serv_tracker.sadd(f'obj:retro_hunt:item:{id}', task_uuid)
def get_retro_hunt_all_item_dates(task_uuid): def get_retro_hunt_all_item_dates(task_uuid):
return r_serv_tracker.zrange(f'tracker:retro_hunt:task:stat:{task_uuid}', 0, -1) return r_serv_tracker.zrange(f'tracker:retro_hunt:task:stat:{task_uuid}', 0, -1)

2
bin/packages/Term.py
View File

@ -25,7 +25,7 @@ import Date
import Item import Item
config_loader = ConfigLoader.ConfigLoader() config_loader = ConfigLoader.ConfigLoader()
r_serv_term = config_loader.get_redis_conn("ARDB_Tracker") r_serv_term = config_loader.get_redis_conn("Kvrocks_DB")
config_loader = None config_loader = None
email_regex = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}' email_regex = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}'

60
var/www/modules/hunter/Flask_hunter.py
View File

@ -44,49 +44,49 @@ hunter = Blueprint('hunter', __name__, template_folder='templates')
@login_read_only @login_read_only
def tracked_menu(): def tracked_menu():
user_id = current_user.get_id() user_id = current_user.get_id()
user_term = Term.get_all_user_tracked_terms(user_id) user_trackers = Tracker.get_user_trackers_metadata(user_id)
global_term = Term.get_all_global_tracked_terms() global_trackers = Tracker.get_global_trackers_metadata()
return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label) return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label)
@hunter.route("/trackers/word") @hunter.route("/trackers/word")
@login_required @login_required
@login_read_only @login_read_only
def tracked_menu_word(): def tracked_menu_word():
filter_type = 'word' tracker_type = 'word'
user_id = current_user.get_id() user_id = current_user.get_id()
user_term = Term.get_all_user_tracked_terms(user_id, filter_type='word') user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type='word')
global_term = Term.get_all_global_tracked_terms(filter_type='word') global_trackers = Tracker.get_global_trackers_metadata(tracker_type='word')
return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type) return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
@hunter.route("/trackers/set") @hunter.route("/trackers/set")
@login_required @login_required
@login_read_only @login_read_only
def tracked_menu_set(): def tracked_menu_set():
filter_type = 'set' tracker_type = 'set'
user_id = current_user.get_id() user_id = current_user.get_id()
user_term = Term.get_all_user_tracked_terms(user_id, filter_type=filter_type) user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type=tracker_type)
global_term = Term.get_all_global_tracked_terms(filter_type=filter_type) global_trackers = Tracker.get_global_trackers_metadata(tracker_type=tracker_type)
return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type) return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
@hunter.route("/trackers/regex") @hunter.route("/trackers/regex")
@login_required @login_required
@login_read_only @login_read_only
def tracked_menu_regex(): def tracked_menu_regex():
filter_type = 'regex' tracker_type = 'regex'
user_id = current_user.get_id() user_id = current_user.get_id()
user_term = Term.get_all_user_tracked_terms(user_id, filter_type=filter_type) user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type=tracker_type)
global_term = Term.get_all_global_tracked_terms(filter_type=filter_type) global_trackers = Tracker.get_global_trackers_metadata(tracker_type=tracker_type)
return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type) return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
@hunter.route("/trackers/yara") @hunter.route("/trackers/yara")
@login_required @login_required
@login_read_only @login_read_only
def tracked_menu_yara(): def tracked_menu_yara():
filter_type = 'yara' tracker_type = 'yara'
user_id = current_user.get_id() user_id = current_user.get_id()
user_term = Term.get_all_user_tracked_terms(user_id, filter_type=filter_type) user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type=tracker_type)
global_term = Term.get_all_global_tracked_terms(filter_type=filter_type) global_trackers = Tracker.get_global_trackers_metadata(tracker_type=tracker_type)
return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type) return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
@hunter.route("/tracker/add", methods=['GET', 'POST']) @hunter.route("/tracker/add", methods=['GET', 'POST'])
@ -119,7 +119,7 @@ def add_tracked_menu():
galaxies_tags = json.loads(galaxies_tags) galaxies_tags = json.loads(galaxies_tags)
except Exception: except Exception:
galaxies_tags = [] galaxies_tags = []
# YARA # # YARA #
if tracker_type == 'yara': if tracker_type == 'yara':
@ -170,8 +170,8 @@ def edit_tracked_menu():
user_id = current_user.get_id() user_id = current_user.get_id()
tracker_uuid = request.args.get('uuid', None) tracker_uuid = request.args.get('uuid', None)
res = Term.check_term_uuid_valid_access(tracker_uuid, user_id) # check if is author or admin res = Tracker.api_is_allowed_to_edit_tracker(tracker_uuid, user_id) # check if is author or admin
if res: # invalid access if res[1] != 200: # invalid access
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
dict_tracker = Tracker.get_tracker_metadata(tracker_uuid, user_id=True, level=True, description=True, tags=True, mails=True, sources=True, webhook=True) dict_tracker = Tracker.get_tracker_metadata(tracker_uuid, user_id=True, level=True, description=True, tags=True, mails=True, sources=True, webhook=True)
@ -209,8 +209,8 @@ def edit_tracked_menu():
def show_tracker(): def show_tracker():
user_id = current_user.get_id() user_id = current_user.get_id()
tracker_uuid = request.args.get('uuid', None) tracker_uuid = request.args.get('uuid', None)
res = Term.check_term_uuid_valid_access(tracker_uuid, user_id) res = Tracker.api_is_allowed_to_edit_tracker(tracker_uuid, user_id)
if res: # invalid access if res[1] != 200: # invalid access
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
date_from = request.args.get('date_from') date_from = request.args.get('date_from')
@ -252,8 +252,8 @@ def show_tracker():
def update_tracker_description(): def update_tracker_description():
user_id = current_user.get_id() user_id = current_user.get_id()
term_uuid = request.form.get('uuid') term_uuid = request.form.get('uuid')
res = Term.check_term_uuid_valid_access(term_uuid, user_id) res = Tracker.api_is_allowed_to_edit_tracker(term_uuid, user_id)
if res: # invalid access if res[1] != 200: # invalid access
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
description = escape( str(request.form.get('description', '')) ) description = escape( str(request.form.get('description', '')) )
Term.replace_tracker_description(term_uuid, description) Term.replace_tracker_description(term_uuid, description)
@ -265,8 +265,8 @@ def update_tracker_description():
def update_tracker_tags(): def update_tracker_tags():
user_id = current_user.get_id() user_id = current_user.get_id()
term_uuid = request.form.get('uuid') term_uuid = request.form.get('uuid')
res = Term.check_term_uuid_valid_access(term_uuid, user_id) res = Tracker.api_is_allowed_to_edit_tracker(term_uuid, user_id)
if res: # invalid access if res[1] != 200: # invalid access
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
tags = request.form.get('tags') tags = request.form.get('tags')
if tags: if tags:
@ -282,8 +282,8 @@ def update_tracker_tags():
def update_tracker_mails(): def update_tracker_mails():
user_id = current_user.get_id() user_id = current_user.get_id()
term_uuid = request.form.get('uuid') term_uuid = request.form.get('uuid')
res = Term.check_term_uuid_valid_access(term_uuid, user_id) res = Tracker.api_is_allowed_to_edit_tracker(term_uuid, user_id)
if res: # invalid access if res[1] != 200: # invalid access
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
mails = request.form.get('mails') mails = request.form.get('mails')
if mails: if mails:

46
var/www/modules/hunter/templates/trackersManagement.html
View File

@ -46,9 +46,11 @@
<div class="col-12 col-lg-10" id="core_content"> <div class="col-12 col-lg-10" id="core_content">
{% if user_trackers %}
<div class="card my-3"> <div class="card my-3">
<div class="card-header"> <div class="card-header">
<h5 class="card-title">Your {{ filter_type }} Trackers</h5> <h5 class="card-title">Your {{ tracker_type }} Trackers</h5>
</div> </div>
<div class="card-body"> <div class="card-body">
<table id="table_user_trackers" class="table table-striped table-bordered"> <table id="table_user_trackers" class="table table-striped table-bordered">
@ -63,17 +65,17 @@
</tr> </tr>
</thead> </thead>
<tbody style="font-size: 15px;"> <tbody style="font-size: 15px;">
{% for dict_uuid in user_term %} {% for dict_uuid in user_trackers %}
<tr> <tr>
<td>{{ dict_uuid['type'] }}</td> <td>{{ dict_uuid['type'] }}</td>
<td> <td>
<span> <span>
<a target="_blank" href="{{ url_for('hunter.show_tracker') }}?uuid={{ dict_uuid['uuid'] }}"> <a target="_blank" href="{{ url_for('hunter.show_tracker') }}?uuid={{ dict_uuid['uuid'] }}">
{% if dict_uuid['term'] %} {% if dict_uuid['tracker'] %}
{% if dict_uuid['term']|length > 256 %} {% if dict_uuid['tracker']|length > 256 %}
{{ dict_uuid['term'][0:256] }}... {{ dict_uuid['tracker'][0:256] }}...
{% else %} {% else %}
{{ dict_uuid['term'] }} {{ dict_uuid['tracker'] }}
{% endif %} {% endif %}
{% endif %} {% endif %}
</a> </a>
@ -88,14 +90,12 @@
</td> </td>
<td> <td>
{% if dict_uuid['first_seen'] %} {% if dict_uuid['first_seen'] %}
{{ dict_uuid['first_seen'][0:4] }}/{{ dict_uuid['first_seen'][4:6] }}/ {{ dict_uuid['first_seen'][0:4] }}/{{ dict_uuid['first_seen'][4:6] }}/{{ dict_uuid['first_seen'][6:8] }}
{{ dict_uuid['first_seen'][6:8] }}
{% endif %} {% endif %}
</td> </td>
<td> <td>
{% if dict_uuid['last_seen'] %} {% if dict_uuid['last_seen'] %}
{{ dict_uuid['last_seen'][0:4] }}/{{ dict_uuid['last_seen'][4:6] }}/ {{ dict_uuid['last_seen'][0:4] }}/{{ dict_uuid['last_seen'][4:6] }}/{{ dict_uuid['last_seen'][6:8] }}
{{ dict_uuid['last_seen'][6:8] }}
{% endif %} {% endif %}
</td> </td>
<td> <td>
@ -111,9 +111,11 @@
</div> </div>
</div> </div>
{% endif %}
<div class="card my-3"> <div class="card my-3">
<div class="card-header"> <div class="card-header">
<h5 class="card-title">Global {{ filter_type }} Trackers</h5> <h5 class="card-title">Global {{ tracker_type }} Trackers</h5>
</div> </div>
<div class="card-body"> <div class="card-body">
<table id="table_global_trackers" class="table table-striped table-bordered"> <table id="table_global_trackers" class="table table-striped table-bordered">
@ -128,17 +130,17 @@
</tr> </tr>
</thead> </thead>
<tbody style="font-size: 15px;"> <tbody style="font-size: 15px;">
{% for dict_uuid in global_term %} {% for dict_uuid in global_trackers %}
<tr> <tr>
<td>{{ dict_uuid['type'] }}</td> <td>{{ dict_uuid['type'] }}</td>
<td> <td>
<span> <span>
<a target="_blank" href="{{ url_for('hunter.show_tracker') }}?uuid={{ dict_uuid['uuid'] }}"> <a target="_blank" href="{{ url_for('hunter.show_tracker') }}?uuid={{ dict_uuid['uuid'] }}">
{% if dict_uuid['term'] %} {% if dict_uuid['tracker'] %}
{% if dict_uuid['term']|length > 256 %} {% if dict_uuid['tracker']|length > 256 %}
{{ dict_uuid['term'][0:256] }}... {{ dict_uuid['tracker'][0:256] }}...
{% else %} {% else %}
{{ dict_uuid['term'] }} {{ dict_uuid['tracker'] }}
{% endif %} {% endif %}
{% endif %} {% endif %}
</a> </a>
@ -153,14 +155,12 @@
</td> </td>
<td> <td>
{% if dict_uuid['first_seen'] %} {% if dict_uuid['first_seen'] %}
{{ dict_uuid['first_seen'][0:4] }}/{{ dict_uuid['first_seen'][4:6] }}/ {{ dict_uuid['first_seen'][0:4] }}/{{ dict_uuid['first_seen'][4:6] }}/{{ dict_uuid['first_seen'][6:8] }}
{{ dict_uuid['first_seen'][6:8] }}
{% endif %} {% endif %}
</td> </td>
<td> <td>
{% if dict_uuid['last_seen'] %} {% if dict_uuid['last_seen'] %}
{{ dict_uuid['last_seen'][0:4] }}/{{ dict_uuid['last_seen'][4:6] }}/ {{ dict_uuid['last_seen'][0:4] }}/{{ dict_uuid['last_seen'][4:6] }}/{{ dict_uuid['last_seen'][6:8] }}
{{ dict_uuid['last_seen'][6:8] }}
{% endif %} {% endif %}
</td> </td>
@ -190,12 +190,12 @@
<script> <script>
$(document).ready(function () { $(document).ready(function () {
$("#page-Tracker").addClass("active"); $("#page-Tracker").addClass("active");
$("#nav_tracker_{{filter_type}}").addClass("active"); $("#nav_tracker_{{tracker_type}}").addClass("active");
{% for dict_uuid in user_term %} {% for dict_uuid in user_trackers %}
sparkline("sparklines_{{ dict_uuid['uuid'] }}", {{ dict_uuid['sparkline'] }}, {height: 40}); sparkline("sparklines_{{ dict_uuid['uuid'] }}", {{ dict_uuid['sparkline'] }}, {height: 40});
{% endfor %} {% endfor %}
{% for dict_uuid in global_term %} {% for dict_uuid in global_trackers %}
sparkline("sparklines_{{ dict_uuid['uuid'] }}", {{ dict_uuid['sparkline'] }}, {height: 40}); sparkline("sparklines_{{ dict_uuid['uuid'] }}", {{ dict_uuid['sparkline'] }}, {height: 40});
{% endfor %} {% endfor %}