From e42ec48380d98627fd2397ff1b328cbd219c4e99 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 13 Nov 2019 16:46:28 +0100 Subject: [PATCH] chg: [Correlation Graph] add correlation by pgp, cryptocurrency, decoded and item + request correlation by type(pgp, ...) + objects(domain, paste) --- bin/lib/Correlate_object.py | 158 ++++++++++-------- bin/lib/Decoded.py | 33 +++- bin/lib/Domain.py | 26 +-- bin/packages/Correlation.py | 26 +++ bin/packages/Item.py | 26 +-- .../correlation/show_correlation.html | 2 +- 6 files changed, 167 insertions(+), 104 deletions(-) diff --git a/bin/lib/Correlate_object.py b/bin/lib/Correlate_object.py index 963a0368..eb8dbdc6 100755 --- a/bin/lib/Correlate_object.py +++ b/bin/lib/Correlate_object.py @@ -16,11 +16,30 @@ import Domain sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) import Pgp import Cryptocurrency +import Item config_loader = ConfigLoader.ConfigLoader() r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") config_loader = None + +def get_object_correlation(object_type, value, correlation_names, correlation_objects, requested_correl_type=None): + if object_type == 'domain': + return Domain.get_domain_all_correlation(value, correlation_names=correlation_names) + elif object_type == 'paste': + return Item.get_item_all_correlation(value, correlation_names=correlation_names) + elif object_type == 'decoded': + return Decoded.get_decoded_correlated_object(value, correlation_objects) + elif object_type == 'pgp': + return Pgp.pgp.get_correlation_all_object(requested_correl_type, value, correlation_objects=correlation_objects) + elif object_type == 'cryptocurrency': + return Cryptocurrency.cryptocurrency.get_correlation_all_object(requested_correl_type, value, correlation_objects=correlation_objects) + + return {} + + + + def get_correlation_node_icon(correlation_name, correlation_type=None, value=None): ''' Used in UI Graph. @@ -117,35 +136,6 @@ def get_item_url(correlation_name, value, correlation_type=None): url = url_for(endpoint, paste=value) return url -# # TODO: refractor -# # get object description, return dict, used by correlation -# def get_object_desc(object_type, item_value, correlation_name, correlation_type=None): -# if object_type=="domain": -# return Domain.get_object_desc(item_value) -# if object_type=="correlation": -# return Domain.get_object_desc(item_value) -# {"name": self.correlation_name, "type": correlation_type, "id": correlation_value, "object": correl_object} -# -# -# # # TODO: sanithyse dict_correlation_to_check -# def get_object_correlation(object, object_value, mode, nb_max_elem=400, dict_correlation_to_check=[], depth_limit=1): -# ''' -# Return all correlation of a given item id. -# -# :param l_items_to_correlate: list of dict -# :type l_items_to_correlate: list -# :param mode: correlation mode -# mode == intersection, union -# union: show all related objects -# intersection: show only direct correlation -# :type mode: str -# :param nb_max_elem: max nb of nodes -# :type nb_max_elem: int, optional -# -# -# ''' -# dict_item_desc = {} -# dict_correlation = object.get_correlation(value, dict_correlation_to_check) def create_graph_links(links_set): graph_links_list = [] @@ -168,21 +158,25 @@ def create_graph_nodes(nodes_set, root_node_id): return graph_nodes_list def create_node_id(correlation_name, value, correlation_type=''): + if correlation_type is None: + correlation_type = '' return '{};{};{}'.format(correlation_name, correlation_type, value) -def get_graph_node_domain_correlation(domain, mode, max_nodes=50): + +# # TODO: filter by correlation type => bitcoin, mail, ... +def get_graph_node_object_correlation(object_type, domain, mode, correlation_names, correlation_objects, max_nodes=300, requested_correl_type=None): links = set() nodes = set() - root_node_id = create_node_id('domain', domain) + root_node_id = create_node_id(object_type, domain, requested_correl_type) nodes.add(root_node_id) - domain_correlation = Domain.get_domain_all_correlation(domain) - for correl in domain_correlation: + root_correlation = get_object_correlation(object_type, domain, correlation_names, correlation_objects, requested_correl_type=requested_correl_type) + for correl in root_correlation: if correl in ('pgp', 'cryptocurrency'): - for correl_type in domain_correlation[correl]: - for correl_val in domain_correlation[correl][correl_type]: + for correl_type in root_correlation[correl]: + for correl_val in root_correlation[correl][correl_type]: # add correlation correl_node_id = create_node_id(correl, correl_val, correl_type) @@ -193,33 +187,28 @@ def get_graph_node_domain_correlation(domain, mode, max_nodes=50): nodes.add(correl_node_id) links.add((root_node_id, correl_node_id)) - # get PGP correlation - if correl=='pgp': - res = Pgp.pgp.get_correlation_obj_domain(correl_val, correlation_type=correl_type) # change function for item ? - # get Cryptocurrency correlation - else: - res = Cryptocurrency.cryptocurrency.get_correlation_obj_domain(correl_val, correlation_type=correl_type) - - # inter mode + # get second correlation + res = get_object_correlation(correl, correl_val, correlation_names, correlation_objects, requested_correl_type=correl_type) if res: - for correl_key_val in res: - #filter root domain - if correl_key_val == domain: - continue + for corr_obj in res: + for correl_key_val in res[corr_obj]: + #filter root domain + if correl_key_val == domain: + continue - if len(nodes) > max_nodes: - break - new_corel_1 = create_node_id('domain', correl_key_val) - new_corel_2 = create_node_id(correl, correl_val, correl_type) - nodes.add(new_corel_1) - nodes.add(new_corel_2) - links.add((new_corel_1, new_corel_2)) + if len(nodes) > max_nodes: + break + new_corel_1 = create_node_id(corr_obj, correl_key_val) + new_corel_2 = create_node_id(correl, correl_val, correl_type) + nodes.add(new_corel_1) + nodes.add(new_corel_2) + links.add((new_corel_1, new_corel_2)) - if mode=="inter": - nodes.add(correl_node_id) - links.add((root_node_id, correl_node_id)) - if correl=='decoded': - for correl_val in domain_correlation[correl]: + if mode=="inter": + nodes.add(correl_node_id) + links.add((root_node_id, correl_node_id)) + if correl in ('decoded', 'domain', 'paste'): + for correl_val in root_correlation[correl]: correl_node_id = create_node_id(correl, correl_val) if mode=="union": @@ -228,24 +217,45 @@ def get_graph_node_domain_correlation(domain, mode, max_nodes=50): nodes.add(correl_node_id) links.add((root_node_id, correl_node_id)) - res = Decoded.get_decoded_domain_item(correl_val) + res = get_object_correlation(correl, correl_val, correlation_names, correlation_objects) if res: - for correl_key_val in res: - #filter root domain - if correl_key_val == domain: - continue + for corr_obj in res: + if corr_obj in ('decoded', 'domain', 'paste'): + for correl_key_val in res[corr_obj]: + #filter root domain + if correl_key_val == domain: + continue - if len(nodes) > max_nodes: - break - new_corel_1 = create_node_id('domain', correl_key_val) - new_corel_2 = create_node_id(correl, correl_val) - nodes.add(new_corel_1) - nodes.add(new_corel_2) - links.add((new_corel_1, new_corel_2)) + if len(nodes) > max_nodes: + break + new_corel_1 = create_node_id(corr_obj, correl_key_val) + new_corel_2 = create_node_id(correl, correl_val) + nodes.add(new_corel_1) + nodes.add(new_corel_2) + links.add((new_corel_1, new_corel_2)) - if mode=="inter": - nodes.add(correl_node_id) - links.add((root_node_id, correl_node_id)) + if mode=="inter": + nodes.add(correl_node_id) + links.add((root_node_id, correl_node_id)) + + if corr_obj in ('pgp', 'cryptocurrency'): + for correl_key_type in res[corr_obj]: + for correl_key_val in res[corr_obj][correl_key_type]: + #filter root domain + if correl_key_val == domain: + continue + + if len(nodes) > max_nodes: + break + new_corel_1 = create_node_id(corr_obj, correl_key_val, correl_key_type) + new_corel_2 = create_node_id(correl, correl_val) + nodes.add(new_corel_1) + nodes.add(new_corel_2) + links.add((new_corel_1, new_corel_2)) + + if mode=="inter": + nodes.add(correl_node_id) + links.add((root_node_id, correl_node_id)) return {"nodes": create_graph_nodes(nodes, root_node_id), "links": create_graph_links(links)} diff --git a/bin/lib/Decoded.py b/bin/lib/Decoded.py index 295dd14f..1dc73825 100755 --- a/bin/lib/Decoded.py +++ b/bin/lib/Decoded.py @@ -63,15 +63,30 @@ def get_decoded_domain_item(sha1_string): else: return [] +def get_decoded_correlated_object(sha1_string, correlation_objects=[]): + ''' + Retun all correlation of a given sha1. + + :param sha1_string: sha1 + :type sha1_string: str + + :return: a dict of all correlation for a given sha1 + :rtype: dict + ''' + if correlation_objects is None: + correlation_objects = Correlation.get_all_correlation_objects() + decoded_correlation = {} + for correlation_object in correlation_objects: + if correlation_object == 'paste': + res = get_decoded_items_list(sha1_string) + elif correlation_object == 'domain': + res = get_decoded_domain_item(sha1_string) + else: + res = None + if res: + decoded_correlation[correlation_object] = res + return decoded_correlation + def save_domain_decoded(domain, sha1_string): r_serv_metadata.sadd('hash_domain:{}'.format(domain), sha1_string) # domain - hash map r_serv_metadata.sadd('domain_hash:{}'.format(sha1_string), domain) # hash - domain ma - -if __name__ == "__main__": - #sha1_str = '1e4db5adc1334ad2c9762db9ff6b845ee6ddc223' - #res = _get_decoded_items_list(sha1_str) - #print(res) - #print(len(res)) - - res = get_domain_decoded_item('2222222dpg65ioqu.onion') - print(res) diff --git a/bin/lib/Domain.py b/bin/lib/Domain.py index b47d7d66..5267839c 100755 --- a/bin/lib/Domain.py +++ b/bin/lib/Domain.py @@ -209,7 +209,7 @@ def get_domain_decoded(domain): return Decoded.get_domain_decoded_item(domain) -def get_domain_all_correlation(domain, correlation_type=None, get_nb=False): +def get_domain_all_correlation(domain, correlation_names=[], get_nb=False): ''' Retun all correlation of a given domain. @@ -219,16 +219,22 @@ def get_domain_all_correlation(domain, correlation_type=None, get_nb=False): :return: a dict of all correlation for a given domain :rtype: dict ''' + if not correlation_names: + correlation_names = Correlation.get_all_correlation_names() domain_correl = {} - res = get_domain_cryptocurrency(domain, get_nb=get_nb) - if res: - domain_correl['cryptocurrency'] = res - res = get_domain_pgp(domain, get_nb=get_nb) - if res: - domain_correl['pgp'] = res - res = get_domain_decoded(domain) - if res: - domain_correl['decoded'] = res + for correlation_name in correlation_names: + if correlation_name=='cryptocurrency': + res = get_domain_cryptocurrency(domain, get_nb=get_nb) + elif correlation_name=='pgp': + res = get_domain_pgp(domain, get_nb=get_nb) + elif correlation_name=='decoded': + res = get_domain_decoded(domain) + else: + res = None + # add correllation to dict + if res: + domain_correl[correlation_name] = res + return domain_correl diff --git a/bin/packages/Correlation.py b/bin/packages/Correlation.py index 61194907..97f02d2f 100755 --- a/bin/packages/Correlation.py +++ b/bin/packages/Correlation.py @@ -12,6 +12,18 @@ config_loader = ConfigLoader.ConfigLoader() r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") config_loader = None +def get_all_correlation_names(): + ''' + Return a list of all available correlations + ''' + return ['pgp', 'cryptocurrency', 'decoded'] + +def get_all_correlation_objects(): + ''' + Return a list of all correllated objects + ''' + return ['domain', 'paste'] + class Correlation(object): def __init__(self, correlation_name, all_correlation_types): @@ -207,6 +219,20 @@ class Correlation(object): return dict_correlation + def get_correlation_all_object(self, correlation_type, correlation_value, correlation_objects=[]): + if correlation_objects is None: + correlation_objects = get_all_correlation_objects() + correlation_obj = {} + for correlation_object in correlation_objects: + if correlation_object == 'paste': + res = self._get_items(correlation_type, correlation_value) + elif correlation_object == 'domain': + res = self.get_correlation_obj_domain(correlation_value, correlation_type=correlation_type) + else: + res = None + if res: + correlation_obj[correlation_object] = res + return correlation_obj def save_domain_correlation(self, domain, correlation_type, correlation_value): r_serv_metadata.sadd('domain_{}_{}:{}'.format(self.correlation_name, correlation_type, domain), correlation_value) diff --git a/bin/packages/Item.py b/bin/packages/Item.py index a5439db2..04898735 100755 --- a/bin/packages/Item.py +++ b/bin/packages/Item.py @@ -9,6 +9,7 @@ import redis sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) import Date import Tag +import Correlation import Cryptocurrency from Pgp import pgp @@ -159,7 +160,7 @@ def get_item_decoded(item_id): ''' return Decoded.get_item_decoded(item_id) -def get_item_all_correlation(item_id, correlation_type=None, get_nb=False): +def get_item_all_correlation(item_id, correlation_names=[], get_nb=False): ''' Retun all correlation of a given item id. @@ -169,16 +170,21 @@ def get_item_all_correlation(item_id, correlation_type=None, get_nb=False): :return: a dict of all correlation for a item id :rtype: dict ''' + if not correlation_names: + correlation_names = Correlation.get_all_correlation_names() item_correl = {} - res = get_item_cryptocurrency(item_id, get_nb=get_nb) - if res: - item_correl['cryptocurrency'] = res - res = get_item_pgp(item_id, get_nb=get_nb) - if res: - item_correl['pgp'] = res - res = get_item_decoded(item_id) - if res: - item_correl['decoded'] = res + for correlation_name in correlation_names: + if correlation_name=='cryptocurrency': + res = get_item_cryptocurrency(item_id, get_nb=get_nb) + elif correlation_name=='pgp': + res = get_item_pgp(item_id, get_nb=get_nb) + elif correlation_name=='decoded': + res = get_item_decoded(item_id) + else: + res = None + # add correllation to dict + if res: + item_correl[correlation_name] = res return item_correl diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index ab5f25eb..352c9b11 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -23,7 +23,7 @@ } line.link:hover{ stroke: red; - stroke-width: 2px + stroke-width: 2px; } .node { pointer-events: all;