From 19f7d8c1e804c4548131c407b7c11ad0aeef2bff Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 11 May 2020 18:11:38 +0200 Subject: [PATCH] chg: [UI correlation] add username correlation graph --- bin/lib/Correlate_object.py | 38 +++++++++++++------ bin/lib/Domain.py | 13 +++++++ bin/lib/telegram.py | 4 +- bin/packages/Item.py | 13 +++++++ var/www/blueprints/correlation.py | 7 ++++ .../modules/hashDecoded/Flask_hashDecoded.py | 2 +- .../correlation/legend_graph_correlation.html | 14 +++++++ .../correlation/show_correlation.html | 14 +++++-- 8 files changed, 88 insertions(+), 17 deletions(-) diff --git a/bin/lib/Correlate_object.py b/bin/lib/Correlate_object.py index 0be6afe6..479105f0 100755 --- a/bin/lib/Correlate_object.py +++ b/bin/lib/Correlate_object.py @@ -13,6 +13,7 @@ import ConfigLoader import Decoded import Domain import Screenshot +import telegram sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) import Pgp @@ -24,7 +25,7 @@ r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") config_loader = None def is_valid_object_type(object_type): - if object_type in ['domain', 'item', 'image', 'decoded', 'pgp', 'cryptocurrency']: + if object_type in ['domain', 'item', 'image', 'decoded', 'pgp', 'cryptocurrency', 'username']: return True else: return False @@ -33,25 +34,22 @@ def is_valid_object_subtype(object_type, object_subtype): if object_type == 'pgp': return Pgp.pgp.is_valid_obj_subtype(object_subtype) elif object_type == 'cryptocurrency': - return Pgp.pgp.is_valid_obj_subtype(object_subtype) + return Cryptocurrency.cryptocurrency.is_valid_obj_subtype(object_subtype) + elif object_type == 'username': + return telegram.correlation.is_valid_obj_subtype(object_subtype) elif object_subtype == None: return True else: return False - if object_type in ['domain', 'item', 'image', 'decoded', 'pgp', 'cryptocurrency']: - return True - else: - return False - def get_all_objects(): - return ['domain', 'paste', 'pgp', 'cryptocurrency', 'decoded', 'screenshot'] + return ['domain', 'paste', 'pgp', 'cryptocurrency', 'decoded', 'screenshot', 'username'] def get_all_correlation_names(): ''' Return a list of all available correlations ''' - return ['pgp', 'cryptocurrency', 'decoded', 'screenshot'] + return ['pgp', 'cryptocurrency', 'decoded', 'screenshot', 'username'] def get_all_correlation_objects(): ''' @@ -70,6 +68,8 @@ def exist_object(object_type, correlation_id, type_id=None): # => work on object return Pgp.pgp.exist_correlation(type_id, correlation_id) elif object_type == 'cryptocurrency': return Cryptocurrency.cryptocurrency.exist_correlation(type_id, correlation_id) + elif object_type == 'username': + return telegram.correlation.exist_correlation(type_id, correlation_id) elif object_type == 'screenshot' or object_type == 'image': return Screenshot.exist_screenshot(correlation_id) else: @@ -87,6 +87,8 @@ def get_object_metadata(object_type, correlation_id, type_id=None): return Pgp.pgp.get_metadata(type_id, correlation_id) elif object_type == 'cryptocurrency': return Cryptocurrency.cryptocurrency.get_metadata(type_id, correlation_id) + elif object_type == 'username': + return telegram.correlation.get_metadata(type_id, correlation_id) elif object_type == 'screenshot' or object_type == 'image': return Screenshot.get_metadata(correlation_id) @@ -101,6 +103,8 @@ def get_object_correlation(object_type, value, correlation_names=None, correlati return Pgp.pgp.get_correlation_all_object(requested_correl_type, value, correlation_objects=correlation_objects) elif object_type == 'cryptocurrency': return Cryptocurrency.cryptocurrency.get_correlation_all_object(requested_correl_type, value, correlation_objects=correlation_objects) + elif object_type == 'username': + return telegram.correlation.get_correlation_all_object(requested_correl_type, value, correlation_objects=correlation_objects) elif object_type == 'screenshot' or object_type == 'image': return Screenshot.get_screenshot_correlated_object(value, correlation_objects=correlation_objects) return {} @@ -118,6 +122,7 @@ def get_correlation_node_icon(correlation_name, correlation_type=None, value=Non :return: a dictionnary {font awesome class, icon_code} :rtype: dict ''' + icon_class = 'fas' icon_text = '' node_color = "#332288" @@ -147,6 +152,14 @@ def get_correlation_node_icon(correlation_name, correlation_type=None, value=Non else: icon_text = '\uf51e' + elif correlation_name == 'username': + node_color = '#4dffff' + if correlation_type == 'telegram': + icon_class = 'fab' + icon_text = '\uf2c6' + else: + icon_text = '\uf007' + elif correlation_name == 'decoded': node_color = '#88CCEE' print(Decoded.get_decoded_item_type(value)) @@ -196,6 +209,9 @@ def get_item_url(correlation_name, value, correlation_type=None): elif correlation_name == 'cryptocurrency': endpoint = 'correlation.show_correlation' url = url_for(endpoint, object_type="cryptocurrency", type_id=correlation_type, correlation_id=value) + elif correlation_name == 'username': + endpoint = 'correlation.show_correlation' + url = url_for(endpoint, object_type="username", type_id=correlation_type, correlation_id=value) elif correlation_name == 'decoded': endpoint = 'correlation.show_correlation' url = url_for(endpoint, object_type="decoded", correlation_id=value) @@ -285,7 +301,7 @@ def get_graph_node_object_correlation(object_type, root_value, mode, correlation root_correlation = get_object_correlation(object_type, root_value, correlation_names, correlation_objects, requested_correl_type=requested_correl_type) for correl in root_correlation: - if correl in ('pgp', 'cryptocurrency'): + if correl in ('pgp', 'cryptocurrency', 'username'): for correl_type in root_correlation[correl]: for correl_val in root_correlation[correl][correl_type]: @@ -349,7 +365,7 @@ def get_graph_node_object_correlation(object_type, root_value, mode, correlation nodes.add(correl_node_id) links.add((root_node_id, correl_node_id)) - if corr_obj in ('pgp', 'cryptocurrency'): + if corr_obj in ('pgp', 'cryptocurrency', 'username'): for correl_key_type in res[corr_obj]: for correl_key_val in res[corr_obj][correl_key_type]: #filter root value diff --git a/bin/lib/Domain.py b/bin/lib/Domain.py index b8ba49e3..d6c7247f 100755 --- a/bin/lib/Domain.py +++ b/bin/lib/Domain.py @@ -25,6 +25,7 @@ sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader import Correlate_object import Screenshot +import telegram config_loader = ConfigLoader.ConfigLoader() r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") @@ -555,6 +556,16 @@ def get_domain_pgp(domain, currencies_type=None, get_nb=False): ''' return Pgp.pgp.get_domain_correlation_dict(domain, correlation_type=currencies_type, get_nb=get_nb) +def get_domain_username(domain, currencies_type=None, get_nb=False): + ''' + Retun all pgp of a given domain. + + :param domain: crawled domain + :param currencies_type: list of pgp type + :type currencies_type: list, optional + ''' + return telegram.correlation.get_domain_correlation_dict(domain, correlation_type=currencies_type, get_nb=get_nb) + def get_domain_decoded(domain): ''' Retun all decoded item of a given domain. @@ -590,6 +601,8 @@ def get_domain_all_correlation(domain, correlation_names=[], get_nb=False): res = get_domain_cryptocurrency(domain, get_nb=get_nb) elif correlation_name=='pgp': res = get_domain_pgp(domain, get_nb=get_nb) + elif correlation_name=='username': + res = get_domain_username(domain, get_nb=get_nb) elif correlation_name=='decoded': res = get_domain_decoded(domain) elif correlation_name=='screenshot': diff --git a/bin/lib/telegram.py b/bin/lib/telegram.py index ebb842a6..08eafeff 100755 --- a/bin/lib/telegram.py +++ b/bin/lib/telegram.py @@ -15,10 +15,10 @@ config_loader = ConfigLoader.ConfigLoader() r_serv_crawler = config_loader.get_redis_conn("ARDB_Onion") config_loader = None -correlaton = Correlation.Correlation('username', ['telegram']) +correlation = Correlation.Correlation('username', ['telegram']) def save_item_correlation(username, item_id, item_date): - correlaton.save_item_correlation('telegram', username, item_id, item_date) + correlation.save_item_correlation('telegram', username, item_id, item_date) def save_telegram_invite_hash(invite_hash, item_id): r_serv_crawler.sadd('telegram:invite_code', '{};{}'.format(invite_hash, item_id)) diff --git a/bin/packages/Item.py b/bin/packages/Item.py index ad9a41ad..e319d1bf 100755 --- a/bin/packages/Item.py +++ b/bin/packages/Item.py @@ -20,6 +20,7 @@ import ConfigLoader import Correlate_object import Decoded import Screenshot +import telegram config_loader = ConfigLoader.ConfigLoader() # get and sanityze PASTE DIRECTORY @@ -171,6 +172,16 @@ def get_item_pgp(item_id, currencies_type=None, get_nb=False): ''' return Pgp.pgp.get_item_correlation_dict(item_id, correlation_type=currencies_type, get_nb=get_nb) +def get_item_username(item_id, currencies_type=None, get_nb=False): + ''' + Return all pgp of a given item. + + :param item_id: item id + :param currencies_type: list of cryptocurrencies type + :type currencies_type: list, optional + ''' + return telegram.correlation.get_item_correlation_dict(item_id, correlation_type=currencies_type, get_nb=get_nb) + def get_item_decoded(item_id): ''' Return all pgp of a given item. @@ -207,6 +218,8 @@ def get_item_all_correlation(item_id, correlation_names=[], get_nb=False): res = get_item_cryptocurrency(item_id, get_nb=get_nb) elif correlation_name=='pgp': res = get_item_pgp(item_id, get_nb=get_nb) + elif correlation_name=='username': + res = get_item_username(item_id, get_nb=get_nb) elif correlation_name=='decoded': res = get_item_decoded(item_id) elif correlation_name=='screenshot': diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py index 6096bb52..359ac612 100644 --- a/var/www/blueprints/correlation.py +++ b/var/www/blueprints/correlation.py @@ -25,6 +25,7 @@ import Correlate_object import Domain import Screenshot import btc_ail +import telegram sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) import Cryptocurrency @@ -108,6 +109,9 @@ def get_card_metadata(object_type, correlation_id, type_id=None, expand_card=Fal elif object_type == 'pgp': card_dict["sparkline"] = Pgp.pgp.get_list_nb_previous_correlation_object(type_id, correlation_id, 6) card_dict["icon"] = Correlate_object.get_correlation_node_icon(object_type, type_id) + elif object_type == 'username': + card_dict["sparkline"] = telegram.correlation.get_list_nb_previous_correlation_object(type_id, correlation_id, 6) + card_dict["icon"] = Correlate_object.get_correlation_node_icon(object_type, type_id) elif object_type == 'decoded': card_dict["sparkline"] = Decoded.get_list_nb_previous_hash(correlation_id, 6) card_dict["icon"] = Correlate_object.get_correlation_node_icon(object_type, value=correlation_id) @@ -149,6 +153,9 @@ def show_correlation(): correl_option = request.form.get('PgpCheck') if correl_option: correlation_names.append('pgp') + correl_option = request.form.get('UsernameCheck') + if correl_option: + correlation_names.append('username') correl_option = request.form.get('DecodedCheck') if correl_option: correlation_names.append('decoded') diff --git a/var/www/modules/hashDecoded/Flask_hashDecoded.py b/var/www/modules/hashDecoded/Flask_hashDecoded.py index 5c92b160..dd1c341a 100644 --- a/var/www/modules/hashDecoded/Flask_hashDecoded.py +++ b/var/www/modules/hashDecoded/Flask_hashDecoded.py @@ -146,7 +146,7 @@ def get_icon_text(correlation_type, type_id): icon_text = '\uf42e' else: icon_text = '\uf51e' - elif correlation_type == 'cryptocurrency': + elif correlation_type == 'username': if type_id == 'telegram': icon_text = '\uf2c6' return icon_text diff --git a/var/www/templates/correlation/legend_graph_correlation.html b/var/www/templates/correlation/legend_graph_correlation.html index 45cee03c..15fee4d9 100644 --- a/var/www/templates/correlation/legend_graph_correlation.html +++ b/var/www/templates/correlation/legend_graph_correlation.html @@ -13,6 +13,9 @@ Pgp: + + Username: + Domain: @@ -139,6 +142,17 @@ mail + +
+ + + + + + + telegram +
+
diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index 419c1e04..1ca33d9a 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -95,6 +95,8 @@ {% include 'correlation/metadata_card_pgp.html' %} {% elif dict_object["object_type"] == "cryptocurrency" %} {% include 'correlation/metadata_card_cryptocurrency.html' %} + {% elif dict_object["object_type"] == "username" %} + {% include 'correlation/metadata_card_username.html' %} {% elif dict_object["object_type"] == "decoded" %} {% include 'correlation/metadata_card_decoded.html' %} {% elif dict_object["object_type"] == "domain" %} @@ -112,9 +114,11 @@
Graph - {% with obj_type=dict_object["object_type"], obj_id=dict_object["correlation_id"], obj_subtype=dict_object["metadata"]["type_id"],obj_lvl=1%} - {% include 'import_export/block_add_user_object_to_export.html' %} - {% endwith %} + {% if dict_object["object_type"] != "username" %} + {% with obj_type=dict_object["object_type"], obj_id=dict_object["correlation_id"], obj_subtype=dict_object["metadata"]["type_id"],obj_lvl=1%} + {% include 'import_export/block_add_user_object_to_export.html' %} + {% endwith %} + {% endif %}
+
+ + +