mirror of https://github.com/CIRCL/AIL-framework
chg: [correlation] tags all objects
Terrtia
parent
3b1fc0fd2a
commit
f45f9e5836
|
|
@ -118,6 +118,33 @@ def delete_obj_correlation(obj1_type, subtype1, obj1_id, obj2_type, subtype2, ob
|
|||
r_metadata.srem(f'correlation:obj:{obj1_type}:{subtype1}:{obj2_type}:{obj1_id}', f'{subtype2}:{obj2_id}')
|
||||
r_metadata.srem(f'correlation:obj:{obj2_type}:{subtype2}:{obj1_type}:{obj2_id}', f'{subtype1}:{obj1_id}')
|
||||
|
||||
# # bypass max result/objects ???
|
||||
# def get_correlation_depht(obj_type, subtype, obj_id, filter_types=[], level=1, nb_max=300):
|
||||
# objs = set()
|
||||
# _get_correlation_depht(objs, obj_type, subtype, obj_id, filter_types, level, nb_max)
|
||||
# return objs
|
||||
#
|
||||
# def _get_correlation_depht(objs, obj_type, subtype, obj_id, filter_types, level, nb_max, previous_str_obj=''):
|
||||
# obj_str_id = get_obj_str_id(obj_type, subtype, obj_id)
|
||||
# objs.add(obj_str_id)
|
||||
#
|
||||
# obj_correlations = get_correlations(obj_type, subtype, obj_id, filter_types=filter_types)
|
||||
# for correl_type in obj_correlations:
|
||||
# for str_obj in obj_correlations[correl_type]:
|
||||
# subtype2, obj2_id = str_obj.split(':', 1)
|
||||
# obj2_str_id = get_obj_str_id(correl_type, subtype2, obj2_id)
|
||||
#
|
||||
# if obj2_str_id == previous_str_obj:
|
||||
# continue
|
||||
#
|
||||
# if len(nodes) > nb_max:
|
||||
# break
|
||||
# objs.add(obj2_str_id)
|
||||
#
|
||||
# if level > 0:
|
||||
# next_level = level - 1
|
||||
# _get_correlation_depht(objs, correl_type, subtype2, obj2_id, filter_types, next_level, nb_max,
|
||||
# previous_str_obj=obj_str_id)
|
||||
|
||||
def get_obj_str_id(obj_type, subtype, obj_id):
|
||||
if subtype is None:
|
||||
|
|
|
|||
|
|
@ -213,11 +213,11 @@ class AbstractObject(ABC):
|
|||
"""
|
||||
return get_correlations(self.type, self.subtype, self.id, filter_types=[obj_type])
|
||||
|
||||
def get_correlations(self):
|
||||
def get_correlations(self, filter_types=[]):
|
||||
"""
|
||||
Get object correlations
|
||||
"""
|
||||
return get_correlations(self.type, self.subtype, self.id)
|
||||
return get_correlations(self.type, self.subtype, self.id, filter_types=filter_types)
|
||||
|
||||
def get_nb_correlation(self, correl_type):
|
||||
return get_nb_correlation_by_correl_type(self.type, self.get_subtype(r_str=True), self.id, correl_type)
|
||||
|
|
|
|||
|
|
@ -273,27 +273,31 @@ def get_obj_correlations(obj_type, subtype, obj_id):
|
|||
obj = get_object(obj_type, subtype, obj_id)
|
||||
return obj.get_correlations()
|
||||
|
||||
def _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, lvl=0):
|
||||
if lvl > 0 and (obj_type, subtype, obj_id) not in objs: # Avoid looking for the same correlation
|
||||
objs.add((obj_type, subtype, obj_id))
|
||||
lvl = lvl - 1
|
||||
obj = get_object(obj_type, subtype, obj_id)
|
||||
correlations = obj.get_correlations()
|
||||
# print('--------------------------')
|
||||
# print( obj_id, correlations)
|
||||
# print(lvl)
|
||||
# print('--------------------------')
|
||||
for obj2_type in correlations:
|
||||
for str_obj in correlations[obj2_type]:
|
||||
obj2_subtype, obj2_id = str_obj.split(':', 1)
|
||||
_get_obj_correlations_objs(objs, obj2_type, obj2_subtype, obj2_id, lvl=lvl)
|
||||
# print(len(objs))
|
||||
objs.add((obj_type, subtype, obj_id))
|
||||
def _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max):
|
||||
if len(objs) < nb_max or nb_max == -1:
|
||||
if lvl == 0:
|
||||
objs.add((obj_type, subtype, obj_id))
|
||||
|
||||
elif lvl > 0 and (obj_type, subtype, obj_id) not in objs: # Avoid looking for the same correlation
|
||||
objs.add((obj_type, subtype, obj_id))
|
||||
obj = get_object(obj_type, subtype, obj_id)
|
||||
correlations = obj.get_correlations(filter_types=filter_types)
|
||||
lvl = lvl - 1
|
||||
for obj2_type in correlations:
|
||||
for str_obj in correlations[obj2_type]:
|
||||
obj2_subtype, obj2_id = str_obj.split(':', 1)
|
||||
_get_obj_correlations_objs(objs, obj2_type, obj2_subtype, obj2_id, filter_types, lvl, nb_max)
|
||||
|
||||
def get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=[], lvl=0, nb_max=300):
|
||||
objs = set()
|
||||
_get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max)
|
||||
return objs
|
||||
|
||||
def get_obj_correlations_objs(obj_type, subtype, obj_id, lvl=0):
|
||||
objs = set()
|
||||
_get_obj_correlations_objs(objs, obj_type, subtype, obj_id, lvl=lvl)
|
||||
def obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=[], lvl=0, nb_max=300):
|
||||
objs = get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=filter_types, lvl=lvl, nb_max=nb_max)
|
||||
for obj_tuple in objs:
|
||||
obj1_type, subtype1, id1 = obj_tuple
|
||||
add_obj_tags(obj1_type, subtype1, id1, tags)
|
||||
return objs
|
||||
|
||||
################################################################################
|
||||
|
|
|
|||
|
|
@ -24,22 +24,17 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
# Import Project packages
|
||||
##################################
|
||||
from lib.objects import ail_objects
|
||||
from lib import Tag
|
||||
|
||||
bootstrap_label = Flask_config.bootstrap_label
|
||||
vt_enabled = Flask_config.vt_enabled
|
||||
|
||||
# ============ BLUEPRINT ============
|
||||
correlation = Blueprint('correlation', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/correlation'))
|
||||
correlation = Blueprint('correlation', __name__,
|
||||
template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/correlation'))
|
||||
|
||||
# ============ VARIABLES ============
|
||||
|
||||
######
|
||||
### graph_line_json
|
||||
### 'hashDecoded.pgpdump_graph_line_json'
|
||||
### 'hashDecoded.cryptocurrency_graph_line_json'
|
||||
###
|
||||
######
|
||||
|
||||
# ============ FUNCTIONS ============
|
||||
|
||||
def sanitise_graph_mode(graph_mode):
|
||||
|
|
@ -53,12 +48,12 @@ def sanitise_nb_max_nodes(nb_max_nodes):
|
|||
nb_max_nodes = int(nb_max_nodes)
|
||||
if nb_max_nodes < 2:
|
||||
nb_max_nodes = 300
|
||||
except:
|
||||
except (TypeError, ValueError):
|
||||
nb_max_nodes = 300
|
||||
return nb_max_nodes
|
||||
|
||||
# ============= ROUTES ==============
|
||||
@correlation.route('/correlation/show', methods=['GET', 'POST']) # GET + POST
|
||||
@correlation.route('/correlation/show', methods=['GET', 'POST'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def show_correlation():
|
||||
|
|
@ -106,7 +101,7 @@ def show_correlation():
|
|||
|
||||
# redirect to keep history and bookmark
|
||||
return redirect(url_for('correlation.show_correlation', type=object_type, subtype=subtype, id=obj_id, mode=mode,
|
||||
max_nodes=max_nodes, filter=filter_types))
|
||||
max_nodes=max_nodes, filter=filter_types))
|
||||
|
||||
# request.method == 'GET'
|
||||
else:
|
||||
|
|
@ -120,13 +115,9 @@ def show_correlation():
|
|||
|
||||
filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(','))
|
||||
|
||||
# # TODO: remove me, rename screenshot to image
|
||||
if obj_type == 'image':
|
||||
obj_type = 'screenshot'
|
||||
|
||||
# check if obj_id exist
|
||||
if not ail_objects.exists_obj(obj_type, subtype, obj_id):
|
||||
abort(404) # return 404
|
||||
return abort(404)
|
||||
# object exist
|
||||
else:
|
||||
dict_object = {"object_type": obj_type,
|
||||
|
|
@ -138,7 +129,8 @@ def show_correlation():
|
|||
if subtype:
|
||||
dict_object["metadata"]['type_id'] = subtype
|
||||
dict_object["metadata_card"] = ail_objects.get_object_card_meta(obj_type, subtype, obj_id, related_btc=related_btc)
|
||||
return render_template("show_correlation.html", dict_object=dict_object, bootstrap_label=bootstrap_label)
|
||||
return render_template("show_correlation.html", dict_object=dict_object, bootstrap_label=bootstrap_label,
|
||||
tags_selector_data=Tag.get_tags_selector_data())
|
||||
|
||||
@correlation.route('/correlation/get/description')
|
||||
@login_required
|
||||
|
|
@ -158,7 +150,6 @@ def get_description():
|
|||
else:
|
||||
return jsonify({})
|
||||
|
||||
|
||||
# check if correlation_id exist
|
||||
# # TODO: return error json
|
||||
if not ail_objects.exists_obj(object_type, type_id, correlation_id):
|
||||
|
|
@ -179,11 +170,48 @@ def graph_node_json():
|
|||
|
||||
filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(','))
|
||||
|
||||
# # TODO: remove me, rename screenshot
|
||||
if obj_type == 'image':
|
||||
obj_type = 'screenshot'
|
||||
|
||||
json_graph = ail_objects.get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=filter_types, max_nodes=max_nodes, level=2, flask_context=True)
|
||||
#json_graph = Correlate_object.get_graph_node_object_correlation(obj_type, obj_id, 'union', correlation_names, correlation_objects, requested_correl_type=subtype, max_nodes=max_nodes)
|
||||
return jsonify(json_graph)
|
||||
|
||||
@correlation.route('/correlation/tags/add', methods=['POST'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def correlation_tags_add():
|
||||
obj_id = request.form.get('tag_obj_id')
|
||||
subtype = request.form.get('tag_subtype', '')
|
||||
obj_type = request.form.get('tag_obj_type')
|
||||
nb_max = sanitise_nb_max_nodes(request.form.get('tag_nb_max'))
|
||||
filter_types = ail_objects.sanitize_objs_types(request.form.get('tag_filter', '').split(','))
|
||||
|
||||
if not ail_objects.exists_obj(obj_type, subtype, obj_id):
|
||||
return abort(404)
|
||||
|
||||
# tags
|
||||
taxonomies_tags = request.form.get('taxonomies_tags')
|
||||
if taxonomies_tags:
|
||||
try:
|
||||
taxonomies_tags = json.loads(taxonomies_tags)
|
||||
except Exception:
|
||||
taxonomies_tags = []
|
||||
else:
|
||||
taxonomies_tags = []
|
||||
galaxies_tags = request.form.get('galaxies_tags')
|
||||
if galaxies_tags:
|
||||
try:
|
||||
galaxies_tags = json.loads(galaxies_tags)
|
||||
except Exception:
|
||||
galaxies_tags = []
|
||||
if taxonomies_tags or galaxies_tags:
|
||||
if not Tag.is_valid_tags_taxonomies_galaxy(taxonomies_tags, galaxies_tags):
|
||||
return {'error': 'Invalid tag(s)'}, 400
|
||||
tags = taxonomies_tags + galaxies_tags
|
||||
else:
|
||||
tags = []
|
||||
|
||||
if tags:
|
||||
ail_objects.obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=filter_types, lvl=2, nb_max=nb_max)
|
||||
|
||||
return redirect(url_for('correlation.show_correlation',
|
||||
type=obj_type, subtype=subtype, id=obj_id,
|
||||
filter=",".join(filter_types)))
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@
|
|||
<script src="{{ url_for('static', filename='js/helper.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
|
||||
|
||||
<style>
|
||||
.icon_legend {
|
||||
|
|
@ -253,7 +253,26 @@
|
|||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
|
||||
<div class="card my-2">
|
||||
<div class="card-header bg-light">
|
||||
<h4><i class="fas fa-tags"></i> Tags All Objects</h4>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<form action="{{ url_for('correlation.correlation_tags_add') }}" method='post' onsubmit="SubmitAddTags();">
|
||||
<input type="hidden" id="tag_obj_type" name="tag_obj_type" value="{{ dict_object["object_type"] }}">
|
||||
<input type="hidden" id="tag_subtype" name="tag_subtype" value="{{ dict_object["metadata"]["type_id"] }}">
|
||||
<input type="hidden" id="tag_obj_id" name="tag_obj_id" value="{{ dict_object["correlation_id"] }}">
|
||||
<input type="hidden" id="tag_nb_max" name="tag_nb_max" value="{{dict_object["max_nodes"]}}">
|
||||
<input type="hidden" id="filter" name="tag_filter" value="{{dict_object["filter_str"]}}">
|
||||
{% include 'tags/block_tags_selector.html' %}
|
||||
<button class="btn btn-primary mt-2">
|
||||
<i class="fas fa-tag"></i> Add Tags
|
||||
</button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -292,6 +311,14 @@ function toggle_sidebar(){
|
|||
$('#core_content').addClass('col-lg-10')
|
||||
}
|
||||
}
|
||||
|
||||
function SubmitAddTags() {
|
||||
var tags = ltags.getValue();
|
||||
var tagsgalaxy = ltagsgalaxies.getValue();
|
||||
$('#ltags').val(tags);
|
||||
$('#ltagsgalaxies').val(tagsgalaxy);
|
||||
return true;
|
||||
}
|
||||
</script>
|
||||
|
||||
<script>
|
||||
|
|
|
|||
Loading…
Reference in New Issue