From f45f9e583607aa04cedda3f867a38d55e0e49718 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 5 Apr 2023 16:09:06 +0200 Subject: [PATCH] chg: [correlation] tags all objects --- bin/lib/correlations_engine.py | 27 +++++++ bin/lib/objects/abstract_object.py | 4 +- bin/lib/objects/ail_objects.py | 42 ++++++----- var/www/blueprints/correlation.py | 72 +++++++++++++------ .../correlation/show_correlation.html | 31 +++++++- 5 files changed, 131 insertions(+), 45 deletions(-) diff --git a/bin/lib/correlations_engine.py b/bin/lib/correlations_engine.py index e1fbeb4e..27307ad4 100755 --- a/bin/lib/correlations_engine.py +++ b/bin/lib/correlations_engine.py @@ -118,6 +118,33 @@ def delete_obj_correlation(obj1_type, subtype1, obj1_id, obj2_type, subtype2, ob r_metadata.srem(f'correlation:obj:{obj1_type}:{subtype1}:{obj2_type}:{obj1_id}', f'{subtype2}:{obj2_id}') r_metadata.srem(f'correlation:obj:{obj2_type}:{subtype2}:{obj1_type}:{obj2_id}', f'{subtype1}:{obj1_id}') +# # bypass max result/objects ??? +# def get_correlation_depht(obj_type, subtype, obj_id, filter_types=[], level=1, nb_max=300): +# objs = set() +# _get_correlation_depht(objs, obj_type, subtype, obj_id, filter_types, level, nb_max) +# return objs +# +# def _get_correlation_depht(objs, obj_type, subtype, obj_id, filter_types, level, nb_max, previous_str_obj=''): +# obj_str_id = get_obj_str_id(obj_type, subtype, obj_id) +# objs.add(obj_str_id) +# +# obj_correlations = get_correlations(obj_type, subtype, obj_id, filter_types=filter_types) +# for correl_type in obj_correlations: +# for str_obj in obj_correlations[correl_type]: +# subtype2, obj2_id = str_obj.split(':', 1) +# obj2_str_id = get_obj_str_id(correl_type, subtype2, obj2_id) +# +# if obj2_str_id == previous_str_obj: +# continue +# +# if len(nodes) > nb_max: +# break +# objs.add(obj2_str_id) +# +# if level > 0: +# next_level = level - 1 +# _get_correlation_depht(objs, correl_type, subtype2, obj2_id, filter_types, next_level, nb_max, +# previous_str_obj=obj_str_id) def get_obj_str_id(obj_type, subtype, obj_id): if subtype is None: diff --git a/bin/lib/objects/abstract_object.py b/bin/lib/objects/abstract_object.py index 7408ba67..0a52618f 100755 --- a/bin/lib/objects/abstract_object.py +++ b/bin/lib/objects/abstract_object.py @@ -213,11 +213,11 @@ class AbstractObject(ABC): """ return get_correlations(self.type, self.subtype, self.id, filter_types=[obj_type]) - def get_correlations(self): + def get_correlations(self, filter_types=[]): """ Get object correlations """ - return get_correlations(self.type, self.subtype, self.id) + return get_correlations(self.type, self.subtype, self.id, filter_types=filter_types) def get_nb_correlation(self, correl_type): return get_nb_correlation_by_correl_type(self.type, self.get_subtype(r_str=True), self.id, correl_type) diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index 4525d033..9cf87eaf 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -273,27 +273,31 @@ def get_obj_correlations(obj_type, subtype, obj_id): obj = get_object(obj_type, subtype, obj_id) return obj.get_correlations() -def _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, lvl=0): - if lvl > 0 and (obj_type, subtype, obj_id) not in objs: # Avoid looking for the same correlation - objs.add((obj_type, subtype, obj_id)) - lvl = lvl - 1 - obj = get_object(obj_type, subtype, obj_id) - correlations = obj.get_correlations() - # print('--------------------------') - # print( obj_id, correlations) - # print(lvl) - # print('--------------------------') - for obj2_type in correlations: - for str_obj in correlations[obj2_type]: - obj2_subtype, obj2_id = str_obj.split(':', 1) - _get_obj_correlations_objs(objs, obj2_type, obj2_subtype, obj2_id, lvl=lvl) - # print(len(objs)) - objs.add((obj_type, subtype, obj_id)) +def _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max): + if len(objs) < nb_max or nb_max == -1: + if lvl == 0: + objs.add((obj_type, subtype, obj_id)) + + elif lvl > 0 and (obj_type, subtype, obj_id) not in objs: # Avoid looking for the same correlation + objs.add((obj_type, subtype, obj_id)) + obj = get_object(obj_type, subtype, obj_id) + correlations = obj.get_correlations(filter_types=filter_types) + lvl = lvl - 1 + for obj2_type in correlations: + for str_obj in correlations[obj2_type]: + obj2_subtype, obj2_id = str_obj.split(':', 1) + _get_obj_correlations_objs(objs, obj2_type, obj2_subtype, obj2_id, filter_types, lvl, nb_max) + +def get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=[], lvl=0, nb_max=300): + objs = set() + _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max) return objs -def get_obj_correlations_objs(obj_type, subtype, obj_id, lvl=0): - objs = set() - _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, lvl=lvl) +def obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=[], lvl=0, nb_max=300): + objs = get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=filter_types, lvl=lvl, nb_max=nb_max) + for obj_tuple in objs: + obj1_type, subtype1, id1 = obj_tuple + add_obj_tags(obj1_type, subtype1, id1, tags) return objs ################################################################################ diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py index ba9c1ba7..7e8a11a6 100644 --- a/var/www/blueprints/correlation.py +++ b/var/www/blueprints/correlation.py @@ -24,22 +24,17 @@ sys.path.append(os.environ['AIL_BIN']) # Import Project packages ################################## from lib.objects import ail_objects +from lib import Tag bootstrap_label = Flask_config.bootstrap_label vt_enabled = Flask_config.vt_enabled # ============ BLUEPRINT ============ -correlation = Blueprint('correlation', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/correlation')) +correlation = Blueprint('correlation', __name__, + template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/correlation')) # ============ VARIABLES ============ -###### -### graph_line_json -### 'hashDecoded.pgpdump_graph_line_json' -### 'hashDecoded.cryptocurrency_graph_line_json' -### -###### - # ============ FUNCTIONS ============ def sanitise_graph_mode(graph_mode): @@ -53,12 +48,12 @@ def sanitise_nb_max_nodes(nb_max_nodes): nb_max_nodes = int(nb_max_nodes) if nb_max_nodes < 2: nb_max_nodes = 300 - except: + except (TypeError, ValueError): nb_max_nodes = 300 return nb_max_nodes # ============= ROUTES ============== -@correlation.route('/correlation/show', methods=['GET', 'POST']) # GET + POST +@correlation.route('/correlation/show', methods=['GET', 'POST']) @login_required @login_read_only def show_correlation(): @@ -106,7 +101,7 @@ def show_correlation(): # redirect to keep history and bookmark return redirect(url_for('correlation.show_correlation', type=object_type, subtype=subtype, id=obj_id, mode=mode, - max_nodes=max_nodes, filter=filter_types)) + max_nodes=max_nodes, filter=filter_types)) # request.method == 'GET' else: @@ -120,13 +115,9 @@ def show_correlation(): filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(',')) - # # TODO: remove me, rename screenshot to image - if obj_type == 'image': - obj_type = 'screenshot' - # check if obj_id exist if not ail_objects.exists_obj(obj_type, subtype, obj_id): - abort(404) # return 404 + return abort(404) # object exist else: dict_object = {"object_type": obj_type, @@ -138,7 +129,8 @@ def show_correlation(): if subtype: dict_object["metadata"]['type_id'] = subtype dict_object["metadata_card"] = ail_objects.get_object_card_meta(obj_type, subtype, obj_id, related_btc=related_btc) - return render_template("show_correlation.html", dict_object=dict_object, bootstrap_label=bootstrap_label) + return render_template("show_correlation.html", dict_object=dict_object, bootstrap_label=bootstrap_label, + tags_selector_data=Tag.get_tags_selector_data()) @correlation.route('/correlation/get/description') @login_required @@ -158,7 +150,6 @@ def get_description(): else: return jsonify({}) - # check if correlation_id exist # # TODO: return error json if not ail_objects.exists_obj(object_type, type_id, correlation_id): @@ -179,11 +170,48 @@ def graph_node_json(): filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(',')) - # # TODO: remove me, rename screenshot - if obj_type == 'image': - obj_type = 'screenshot' - json_graph = ail_objects.get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=filter_types, max_nodes=max_nodes, level=2, flask_context=True) #json_graph = Correlate_object.get_graph_node_object_correlation(obj_type, obj_id, 'union', correlation_names, correlation_objects, requested_correl_type=subtype, max_nodes=max_nodes) return jsonify(json_graph) +@correlation.route('/correlation/tags/add', methods=['POST']) +@login_required +@login_read_only +def correlation_tags_add(): + obj_id = request.form.get('tag_obj_id') + subtype = request.form.get('tag_subtype', '') + obj_type = request.form.get('tag_obj_type') + nb_max = sanitise_nb_max_nodes(request.form.get('tag_nb_max')) + filter_types = ail_objects.sanitize_objs_types(request.form.get('tag_filter', '').split(',')) + + if not ail_objects.exists_obj(obj_type, subtype, obj_id): + return abort(404) + + # tags + taxonomies_tags = request.form.get('taxonomies_tags') + if taxonomies_tags: + try: + taxonomies_tags = json.loads(taxonomies_tags) + except Exception: + taxonomies_tags = [] + else: + taxonomies_tags = [] + galaxies_tags = request.form.get('galaxies_tags') + if galaxies_tags: + try: + galaxies_tags = json.loads(galaxies_tags) + except Exception: + galaxies_tags = [] + if taxonomies_tags or galaxies_tags: + if not Tag.is_valid_tags_taxonomies_galaxy(taxonomies_tags, galaxies_tags): + return {'error': 'Invalid tag(s)'}, 400 + tags = taxonomies_tags + galaxies_tags + else: + tags = [] + + if tags: + ail_objects.obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=filter_types, lvl=2, nb_max=nb_max) + + return redirect(url_for('correlation.show_correlation', + type=obj_type, subtype=subtype, id=obj_id, + filter=",".join(filter_types))) diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index 24be9d8a..636b8848 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -16,7 +16,7 @@ - +