CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [UI correlation] add username correlation graph

pull/497/head
Terrtia 2020-05-11 18:11:38 +02:00
parent 0f7cfe8fb8
commit 19f7d8c1e8
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
8 changed files with 88 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
bin/lib/Correlate_object.py
View File

@ -13,6 +13,7 @@ import ConfigLoader
import Decoded
import Domain
import Screenshot
import telegram
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
import Pgp
@ -24,7 +25,7 @@ r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
config_loader = None
def is_valid_object_type(object_type):
if object_type in ['domain', 'item', 'image', 'decoded', 'pgp', 'cryptocurrency']:
if object_type in ['domain', 'item', 'image', 'decoded', 'pgp', 'cryptocurrency', 'username']:
return True
else:
return False
@ -33,25 +34,22 @@ def is_valid_object_subtype(object_type, object_subtype):
if object_type == 'pgp':
return Pgp.pgp.is_valid_obj_subtype(object_subtype)
elif object_type == 'cryptocurrency':
return Pgp.pgp.is_valid_obj_subtype(object_subtype)
return Cryptocurrency.cryptocurrency.is_valid_obj_subtype(object_subtype)
elif object_type == 'username':
return telegram.correlation.is_valid_obj_subtype(object_subtype)
elif object_subtype == None:
return True
else:
return False
if object_type in ['domain', 'item', 'image', 'decoded', 'pgp', 'cryptocurrency']:
return True
else:
return False
def get_all_objects():
return ['domain', 'paste', 'pgp', 'cryptocurrency', 'decoded', 'screenshot']
return ['domain', 'paste', 'pgp', 'cryptocurrency', 'decoded', 'screenshot', 'username']
def get_all_correlation_names():
'''
Return a list of all available correlations
'''
return ['pgp', 'cryptocurrency', 'decoded', 'screenshot']
return ['pgp', 'cryptocurrency', 'decoded', 'screenshot', 'username']
def get_all_correlation_objects():
'''
@ -70,6 +68,8 @@ def exist_object(object_type, correlation_id, type_id=None): # => work on object
return Pgp.pgp.exist_correlation(type_id, correlation_id)
elif object_type == 'cryptocurrency':
return Cryptocurrency.cryptocurrency.exist_correlation(type_id, correlation_id)
elif object_type == 'username':
return telegram.correlation.exist_correlation(type_id, correlation_id)
elif object_type == 'screenshot' or object_type == 'image':
return Screenshot.exist_screenshot(correlation_id)
else:
@ -87,6 +87,8 @@ def get_object_metadata(object_type, correlation_id, type_id=None):
return Pgp.pgp.get_metadata(type_id, correlation_id)
elif object_type == 'cryptocurrency':
return Cryptocurrency.cryptocurrency.get_metadata(type_id, correlation_id)
elif object_type == 'username':
return telegram.correlation.get_metadata(type_id, correlation_id)
elif object_type == 'screenshot' or object_type == 'image':
return Screenshot.get_metadata(correlation_id)
@ -101,6 +103,8 @@ def get_object_correlation(object_type, value, correlation_names=None, correlati
return Pgp.pgp.get_correlation_all_object(requested_correl_type, value, correlation_objects=correlation_objects)
elif object_type == 'cryptocurrency':
return Cryptocurrency.cryptocurrency.get_correlation_all_object(requested_correl_type, value, correlation_objects=correlation_objects)
elif object_type == 'username':
return telegram.correlation.get_correlation_all_object(requested_correl_type, value, correlation_objects=correlation_objects)
elif object_type == 'screenshot' or object_type == 'image':
return Screenshot.get_screenshot_correlated_object(value, correlation_objects=correlation_objects)
return {}
@ -118,6 +122,7 @@ def get_correlation_node_icon(correlation_name, correlation_type=None, value=Non
:return: a dictionnary {font awesome class, icon_code}
:rtype: dict
'''
icon_class = 'fas'
icon_text = ''
node_color = "#332288"
@ -147,6 +152,14 @@ def get_correlation_node_icon(correlation_name, correlation_type=None, value=Non
else:
icon_text = '\uf51e'
elif correlation_name == 'username':
node_color = '#4dffff'
if correlation_type == 'telegram':
icon_class = 'fab'
icon_text = '\uf2c6'
else:
icon_text = '\uf007'
elif correlation_name == 'decoded':
node_color = '#88CCEE'
print(Decoded.get_decoded_item_type(value))
@ -196,6 +209,9 @@ def get_item_url(correlation_name, value, correlation_type=None):
elif correlation_name == 'cryptocurrency':
endpoint = 'correlation.show_correlation'
url = url_for(endpoint, object_type="cryptocurrency", type_id=correlation_type, correlation_id=value)
elif correlation_name == 'username':
endpoint = 'correlation.show_correlation'
url = url_for(endpoint, object_type="username", type_id=correlation_type, correlation_id=value)
elif correlation_name == 'decoded':
endpoint = 'correlation.show_correlation'
url = url_for(endpoint, object_type="decoded", correlation_id=value)
@ -285,7 +301,7 @@ def get_graph_node_object_correlation(object_type, root_value, mode, correlation
root_correlation = get_object_correlation(object_type, root_value, correlation_names, correlation_objects, requested_correl_type=requested_correl_type)
for correl in root_correlation:
if correl in ('pgp', 'cryptocurrency'):
if correl in ('pgp', 'cryptocurrency', 'username'):
for correl_type in root_correlation[correl]:
for correl_val in root_correlation[correl][correl_type]:
@ -349,7 +365,7 @@ def get_graph_node_object_correlation(object_type, root_value, mode, correlation
nodes.add(correl_node_id)
links.add((root_node_id, correl_node_id))
if corr_obj in ('pgp', 'cryptocurrency'):
if corr_obj in ('pgp', 'cryptocurrency', 'username'):
for correl_key_type in res[corr_obj]:
for correl_key_val in res[corr_obj][correl_key_type]:
#filter root value

13
bin/lib/Domain.py
View File

@ -25,6 +25,7 @@ sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
import Correlate_object
import Screenshot
import telegram
config_loader = ConfigLoader.ConfigLoader()
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
@ -555,6 +556,16 @@ def get_domain_pgp(domain, currencies_type=None, get_nb=False):
'''
return Pgp.pgp.get_domain_correlation_dict(domain, correlation_type=currencies_type, get_nb=get_nb)
def get_domain_username(domain, currencies_type=None, get_nb=False):
'''
Retun all pgp of a given domain.
:param domain: crawled domain
:param currencies_type: list of pgp type
:type currencies_type: list, optional
'''
return telegram.correlation.get_domain_correlation_dict(domain, correlation_type=currencies_type, get_nb=get_nb)
def get_domain_decoded(domain):
'''
Retun all decoded item of a given domain.
@ -590,6 +601,8 @@ def get_domain_all_correlation(domain, correlation_names=[], get_nb=False):
res = get_domain_cryptocurrency(domain, get_nb=get_nb)
elif correlation_name=='pgp':
res = get_domain_pgp(domain, get_nb=get_nb)
elif correlation_name=='username':
res = get_domain_username(domain, get_nb=get_nb)
elif correlation_name=='decoded':
res = get_domain_decoded(domain)
elif correlation_name=='screenshot':

4
bin/lib/telegram.py
View File

@ -15,10 +15,10 @@ config_loader = ConfigLoader.ConfigLoader()
r_serv_crawler = config_loader.get_redis_conn("ARDB_Onion")
config_loader = None
correlaton = Correlation.Correlation('username', ['telegram'])
correlation = Correlation.Correlation('username', ['telegram'])
def save_item_correlation(username, item_id, item_date):
correlaton.save_item_correlation('telegram', username, item_id, item_date)
correlation.save_item_correlation('telegram', username, item_id, item_date)
def save_telegram_invite_hash(invite_hash, item_id):
r_serv_crawler.sadd('telegram:invite_code', '{};{}'.format(invite_hash, item_id))

13
bin/packages/Item.py
View File

@ -20,6 +20,7 @@ import ConfigLoader
import Correlate_object
import Decoded
import Screenshot
import telegram
config_loader = ConfigLoader.ConfigLoader()
# get and sanityze PASTE DIRECTORY
@ -171,6 +172,16 @@ def get_item_pgp(item_id, currencies_type=None, get_nb=False):
'''
return Pgp.pgp.get_item_correlation_dict(item_id, correlation_type=currencies_type, get_nb=get_nb)
def get_item_username(item_id, currencies_type=None, get_nb=False):
'''
Return all pgp of a given item.
:param item_id: item id
:param currencies_type: list of cryptocurrencies type
:type currencies_type: list, optional
'''
return telegram.correlation.get_item_correlation_dict(item_id, correlation_type=currencies_type, get_nb=get_nb)
def get_item_decoded(item_id):
'''
Return all pgp of a given item.
@ -207,6 +218,8 @@ def get_item_all_correlation(item_id, correlation_names=[], get_nb=False):
res = get_item_cryptocurrency(item_id, get_nb=get_nb)
elif correlation_name=='pgp':
res = get_item_pgp(item_id, get_nb=get_nb)
elif correlation_name=='username':
res = get_item_username(item_id, get_nb=get_nb)
elif correlation_name=='decoded':
res = get_item_decoded(item_id)
elif correlation_name=='screenshot':

7
var/www/blueprints/correlation.py
View File

@ -25,6 +25,7 @@ import Correlate_object
import Domain
import Screenshot
import btc_ail
import telegram
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
import Cryptocurrency
@ -108,6 +109,9 @@ def get_card_metadata(object_type, correlation_id, type_id=None, expand_card=Fal
elif object_type == 'pgp':
card_dict["sparkline"] = Pgp.pgp.get_list_nb_previous_correlation_object(type_id, correlation_id, 6)
card_dict["icon"] = Correlate_object.get_correlation_node_icon(object_type, type_id)
elif object_type == 'username':
card_dict["sparkline"] = telegram.correlation.get_list_nb_previous_correlation_object(type_id, correlation_id, 6)
card_dict["icon"] = Correlate_object.get_correlation_node_icon(object_type, type_id)
elif object_type == 'decoded':
card_dict["sparkline"] = Decoded.get_list_nb_previous_hash(correlation_id, 6)
card_dict["icon"] = Correlate_object.get_correlation_node_icon(object_type, value=correlation_id)
@ -149,6 +153,9 @@ def show_correlation():
correl_option = request.form.get('PgpCheck')
if correl_option:
correlation_names.append('pgp')
correl_option = request.form.get('UsernameCheck')
if correl_option:
correlation_names.append('username')
correl_option = request.form.get('DecodedCheck')
if correl_option:
correlation_names.append('decoded')

2
var/www/modules/hashDecoded/Flask_hashDecoded.py
View File

@ -146,7 +146,7 @@ def get_icon_text(correlation_type, type_id):
icon_text = '\uf42e'
else:
icon_text = '\uf51e'
elif correlation_type == 'cryptocurrency':
elif correlation_type == 'username':
if type_id == 'telegram':
icon_text = '\uf2c6'
return icon_text

14
var/www/templates/correlation/legend_graph_correlation.html
View File

@ -13,6 +13,9 @@
<th class="">
Pgp:
</th>
<th class="">
Username:
</th>
<th class="">
Domain:
</th>
@ -139,6 +142,17 @@
mail
</div>
</td>
<td>
<div class="my-1">
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#4dffff"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab" font-size="16px">&#xf2c6;</text>
</g>
</svg>
telegram
</div>
</td>
<td>
<div class="my-1">
<svg height="26" width="26">

14
var/www/templates/correlation/show_correlation.html
View File

@ -95,6 +95,8 @@
{% include 'correlation/metadata_card_pgp.html' %}
{% elif dict_object["object_type"] == "cryptocurrency" %}
{% include 'correlation/metadata_card_cryptocurrency.html' %}
{% elif dict_object["object_type"] == "username" %}
{% include 'correlation/metadata_card_username.html' %}
{% elif dict_object["object_type"] == "decoded" %}
{% include 'correlation/metadata_card_decoded.html' %}
{% elif dict_object["object_type"] == "domain" %}
@ -112,9 +114,11 @@
<div class="card-header">
<i class="fas fa-project-diagram"></i> Graph
<span class="float-right">
{% with obj_type=dict_object["object_type"], obj_id=dict_object["correlation_id"], obj_subtype=dict_object["metadata"]["type_id"],obj_lvl=1%}
{% include 'import_export/block_add_user_object_to_export.html' %}
{% endwith %}
{% if dict_object["object_type"] != "username" %}
{% with obj_type=dict_object["object_type"], obj_id=dict_object["correlation_id"], obj_subtype=dict_object["metadata"]["type_id"],obj_lvl=1%}
{% include 'import_export/block_add_user_object_to_export.html' %}
{% endwith %}
{% endif %}
</span>
<span class="float-right">
<button class="btn btn-primary py-1" onclick="resize_graph();">
@ -166,6 +170,10 @@
<input class="form-check-input" type="checkbox" value="True" id="PgpCheck" name="PgpCheck" {%if "pgp" in dict_object["correlation_names"]%}checked{%endif%}>
<label class="form-check-label" for="PgpCheck">PGP</label>
</div>
<div class="form-check">
<input class="form-check-input" type="checkbox" value="True" id="UsernameCheck" name="UsernameCheck" {%if "username" in dict_object["correlation_names"]%}checked{%endif%}>
<label class="form-check-label" for="UsernameCheck">Username</label>
</div>
<div class="form-check">
<input class="form-check-input" type="checkbox" value="True" id="DomainCheck" name="DomainCheck" {%if "domain" in dict_object["correlation_objects"]%}checked{%endif%}>
<label class="form-check-label" for="DomainCheck">Domain</label>