CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [correlation] tags all objects

pull/594/head
Terrtia 2023-04-05 16:09:06 +02:00
parent 3b1fc0fd2a
commit f45f9e5836
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
5 changed files with 131 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
bin/lib/correlations_engine.py
View File

@ -118,6 +118,33 @@ def delete_obj_correlation(obj1_type, subtype1, obj1_id, obj2_type, subtype2, ob
r_metadata.srem(f'correlation:obj:{obj1_type}:{subtype1}:{obj2_type}:{obj1_id}', f'{subtype2}:{obj2_id}')
r_metadata.srem(f'correlation:obj:{obj2_type}:{subtype2}:{obj1_type}:{obj2_id}', f'{subtype1}:{obj1_id}')
# # bypass max result/objects ???
# def get_correlation_depht(obj_type, subtype, obj_id, filter_types=[], level=1, nb_max=300):
# objs = set()
# _get_correlation_depht(objs, obj_type, subtype, obj_id, filter_types, level, nb_max)
# return objs
#
# def _get_correlation_depht(objs, obj_type, subtype, obj_id, filter_types, level, nb_max, previous_str_obj=''):
# obj_str_id = get_obj_str_id(obj_type, subtype, obj_id)
# objs.add(obj_str_id)
#
# obj_correlations = get_correlations(obj_type, subtype, obj_id, filter_types=filter_types)
# for correl_type in obj_correlations:
# for str_obj in obj_correlations[correl_type]:
# subtype2, obj2_id = str_obj.split(':', 1)
# obj2_str_id = get_obj_str_id(correl_type, subtype2, obj2_id)
#
# if obj2_str_id == previous_str_obj:
# continue
#
# if len(nodes) > nb_max:
# break
# objs.add(obj2_str_id)
#
# if level > 0:
# next_level = level - 1
# _get_correlation_depht(objs, correl_type, subtype2, obj2_id, filter_types, next_level, nb_max,
# previous_str_obj=obj_str_id)
def get_obj_str_id(obj_type, subtype, obj_id):
if subtype is None:

4
bin/lib/objects/abstract_object.py
View File

@ -213,11 +213,11 @@ class AbstractObject(ABC):
"""
return get_correlations(self.type, self.subtype, self.id, filter_types=[obj_type])
def get_correlations(self):
def get_correlations(self, filter_types=[]):
"""
Get object correlations
"""
return get_correlations(self.type, self.subtype, self.id)
return get_correlations(self.type, self.subtype, self.id, filter_types=filter_types)
def get_nb_correlation(self, correl_type):
return get_nb_correlation_by_correl_type(self.type, self.get_subtype(r_str=True), self.id, correl_type)

32
bin/lib/objects/ail_objects.py
View File

@ -273,27 +273,31 @@ def get_obj_correlations(obj_type, subtype, obj_id):
obj = get_object(obj_type, subtype, obj_id)
return obj.get_correlations()
def _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, lvl=0):
if lvl > 0 and (obj_type, subtype, obj_id) not in objs: # Avoid looking for the same correlation
def _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max):
if len(objs) < nb_max or nb_max == -1:
if lvl == 0:
objs.add((obj_type, subtype, obj_id))
elif lvl > 0 and (obj_type, subtype, obj_id) not in objs: # Avoid looking for the same correlation
objs.add((obj_type, subtype, obj_id))
lvl = lvl - 1
obj = get_object(obj_type, subtype, obj_id)
correlations = obj.get_correlations()
# print('--------------------------')
# print( obj_id, correlations)
# print(lvl)
# print('--------------------------')
correlations = obj.get_correlations(filter_types=filter_types)
lvl = lvl - 1
for obj2_type in correlations:
for str_obj in correlations[obj2_type]:
obj2_subtype, obj2_id = str_obj.split(':', 1)
_get_obj_correlations_objs(objs, obj2_type, obj2_subtype, obj2_id, lvl=lvl)
# print(len(objs))
objs.add((obj_type, subtype, obj_id))
_get_obj_correlations_objs(objs, obj2_type, obj2_subtype, obj2_id, filter_types, lvl, nb_max)
def get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=[], lvl=0, nb_max=300):
objs = set()
_get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max)
return objs
def get_obj_correlations_objs(obj_type, subtype, obj_id, lvl=0):
objs = set()
_get_obj_correlations_objs(objs, obj_type, subtype, obj_id, lvl=lvl)
def obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=[], lvl=0, nb_max=300):
objs = get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=filter_types, lvl=lvl, nb_max=nb_max)
for obj_tuple in objs:
obj1_type, subtype1, id1 = obj_tuple
add_obj_tags(obj1_type, subtype1, id1, tags)
return objs
################################################################################

70
var/www/blueprints/correlation.py
View File

@ -24,22 +24,17 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages
##################################
from lib.objects import ail_objects
from lib import Tag
bootstrap_label = Flask_config.bootstrap_label
vt_enabled = Flask_config.vt_enabled
# ============ BLUEPRINT ============
correlation = Blueprint('correlation', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/correlation'))
correlation = Blueprint('correlation', __name__,
template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/correlation'))
# ============ VARIABLES ============
######
### graph_line_json
### 'hashDecoded.pgpdump_graph_line_json'
### 'hashDecoded.cryptocurrency_graph_line_json'
###
######
# ============ FUNCTIONS ============
def sanitise_graph_mode(graph_mode):
@ -53,12 +48,12 @@ def sanitise_nb_max_nodes(nb_max_nodes):
nb_max_nodes = int(nb_max_nodes)
if nb_max_nodes < 2:
nb_max_nodes = 300
except:
except (TypeError, ValueError):
nb_max_nodes = 300
return nb_max_nodes
# ============= ROUTES ==============
@correlation.route('/correlation/show', methods=['GET', 'POST']) # GET + POST
@correlation.route('/correlation/show', methods=['GET', 'POST'])
@login_required
@login_read_only
def show_correlation():
@ -120,13 +115,9 @@ def show_correlation():
filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(','))
# # TODO: remove me, rename screenshot to image
if obj_type == 'image':
obj_type = 'screenshot'
# check if obj_id exist
if not ail_objects.exists_obj(obj_type, subtype, obj_id):
abort(404) # return 404
return abort(404)
# object exist
else:
dict_object = {"object_type": obj_type,
@ -138,7 +129,8 @@ def show_correlation():
if subtype:
dict_object["metadata"]['type_id'] = subtype
dict_object["metadata_card"] = ail_objects.get_object_card_meta(obj_type, subtype, obj_id, related_btc=related_btc)
return render_template("show_correlation.html", dict_object=dict_object, bootstrap_label=bootstrap_label)
return render_template("show_correlation.html", dict_object=dict_object, bootstrap_label=bootstrap_label,
tags_selector_data=Tag.get_tags_selector_data())
@correlation.route('/correlation/get/description')
@login_required
@ -158,7 +150,6 @@ def get_description():
else:
return jsonify({})
# check if correlation_id exist
# # TODO: return error json
if not ail_objects.exists_obj(object_type, type_id, correlation_id):
@ -179,11 +170,48 @@ def graph_node_json():
filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(','))
# # TODO: remove me, rename screenshot
if obj_type == 'image':
obj_type = 'screenshot'
json_graph = ail_objects.get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=filter_types, max_nodes=max_nodes, level=2, flask_context=True)
#json_graph = Correlate_object.get_graph_node_object_correlation(obj_type, obj_id, 'union', correlation_names, correlation_objects, requested_correl_type=subtype, max_nodes=max_nodes)
return jsonify(json_graph)
@correlation.route('/correlation/tags/add', methods=['POST'])
@login_required
@login_read_only
def correlation_tags_add():
obj_id = request.form.get('tag_obj_id')
subtype = request.form.get('tag_subtype', '')
obj_type = request.form.get('tag_obj_type')
nb_max = sanitise_nb_max_nodes(request.form.get('tag_nb_max'))
filter_types = ail_objects.sanitize_objs_types(request.form.get('tag_filter', '').split(','))
if not ail_objects.exists_obj(obj_type, subtype, obj_id):
return abort(404)
# tags
taxonomies_tags = request.form.get('taxonomies_tags')
if taxonomies_tags:
try:
taxonomies_tags = json.loads(taxonomies_tags)
except Exception:
taxonomies_tags = []
else:
taxonomies_tags = []
galaxies_tags = request.form.get('galaxies_tags')
if galaxies_tags:
try:
galaxies_tags = json.loads(galaxies_tags)
except Exception:
galaxies_tags = []
if taxonomies_tags or galaxies_tags:
if not Tag.is_valid_tags_taxonomies_galaxy(taxonomies_tags, galaxies_tags):
return {'error': 'Invalid tag(s)'}, 400
tags = taxonomies_tags + galaxies_tags
else:
tags = []
if tags:
ail_objects.obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=filter_types, lvl=2, nb_max=nb_max)
return redirect(url_for('correlation.show_correlation',
type=obj_type, subtype=subtype, id=obj_id,
filter=",".join(filter_types)))

29
var/www/templates/correlation/show_correlation.html
View File

@ -16,7 +16,7 @@
<script src="{{ url_for('static', filename='js/helper.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
<style>
.icon_legend {
@ -255,6 +255,25 @@
</div>
{% endif %}
<div class="card my-2">
<div class="card-header bg-light">
<h4><i class="fas fa-tags"></i> Tags All Objects</h4>
</div>
<div class="card-body">
<form action="{{ url_for('correlation.correlation_tags_add') }}" method='post' onsubmit="SubmitAddTags();">
<input type="hidden" id="tag_obj_type" name="tag_obj_type" value="{{ dict_object["object_type"] }}">
<input type="hidden" id="tag_subtype" name="tag_subtype" value="{{ dict_object["metadata"]["type_id"] }}">
<input type="hidden" id="tag_obj_id" name="tag_obj_id" value="{{ dict_object["correlation_id"] }}">
<input type="hidden" id="tag_nb_max" name="tag_nb_max" value="{{dict_object["max_nodes"]}}">
<input type="hidden" id="filter" name="tag_filter" value="{{dict_object["filter_str"]}}">
{% include 'tags/block_tags_selector.html' %}
<button class="btn btn-primary mt-2">
<i class="fas fa-tag"></i> Add Tags
</button>
</form>
</div>
</div>
</div>
</div>
</div>
@ -292,6 +311,14 @@ function toggle_sidebar(){
$('#core_content').addClass('col-lg-10')
}
}
function SubmitAddTags() {
var tags = ltags.getValue();
var tagsgalaxy = ltagsgalaxies.getValue();
$('#ltags').val(tags);
$('#ltagsgalaxies').val(tagsgalaxy);
return true;
}
</script>
<script>