From 2e921a12f27653effc0475fdbc6fd05c484d13e8 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 15 Nov 2019 17:22:50 +0100 Subject: [PATCH] chg: [UI] show correlation (pgp. decoded, cryptocurrency) migration --- bin/lib/Correlate_object.py | 20 +- bin/lib/Decoded.py | 45 +- bin/lib/Domain.py | 4 +- bin/packages/Correlation.py | 26 + bin/packages/Date.py | 11 + var/www/blueprints/correlation.py | 55 +- .../modules/hashDecoded/Flask_hashDecoded.py | 197 +++--- .../templates/DaysCorrelation.html | 5 +- .../hashDecoded/templates/hashDecoded.html | 2 +- .../templates/showCorrelation.html | 512 --------------- .../hashDecoded/templates/showHash.html | 587 ------------------ .../correlation/legend_graph_correlation.html | 168 +++++ .../correlation/metadata_card_decoded.html | 186 ++++++ .../correlation/metadata_card_pgp.html | 140 +++++ .../correlation/show_correlation.html | 101 +-- .../crawler/crawler_splash/showDomain.html | 6 +- .../decoded/show_helper_pgpdump.html | 189 +++++- 17 files changed, 948 insertions(+), 1306 deletions(-) delete mode 100644 var/www/modules/hashDecoded/templates/showCorrelation.html delete mode 100644 var/www/modules/hashDecoded/templates/showHash.html create mode 100644 var/www/templates/correlation/legend_graph_correlation.html create mode 100644 var/www/templates/correlation/metadata_card_decoded.html create mode 100644 var/www/templates/correlation/metadata_card_pgp.html diff --git a/bin/lib/Correlate_object.py b/bin/lib/Correlate_object.py index 95c25f3f..afd82379 100755 --- a/bin/lib/Correlate_object.py +++ b/bin/lib/Correlate_object.py @@ -38,13 +38,13 @@ def get_object_metadata(object_type, correlation_id, type_id=None): if object_type == 'domain': return Domain.Domain(correlation_id).get_domain_metadata() elif object_type == 'paste': - return None + return {} elif object_type == 'decoded': - return Decoded.get_decoded_metadata(correlation_id) + return Decoded.get_decoded_metadata(correlation_id, nb_seen=True, size=True) elif object_type == 'pgp': - return Pgp.pgp._get_metadata(type_id, correlation_id) + return Pgp.pgp.get_metadata(type_id, correlation_id) elif object_type == 'cryptocurrency': - return Cryptocurrency.cryptocurrency._get_metadata(type_id, correlation_id) + return Cryptocurrency.cryptocurrency.get_metadata(type_id, correlation_id) def get_object_correlation(object_type, value, correlation_names, correlation_objects, requested_correl_type=None): if object_type == 'domain': @@ -140,14 +140,14 @@ def get_item_url(correlation_name, value, correlation_type=None): ''' url = '#' if correlation_name == "pgp": - endpoint = 'hashDecoded.show_pgpdump' - url = url_for(endpoint, type_id=correlation_type, key_id=value) + endpoint = 'correlation.show_correlation' + url = url_for(endpoint, object_type="pgp", type_id=correlation_type, correlation_id=value) elif correlation_name == 'cryptocurrency': - endpoint = 'hashDecoded.show_cryptocurrency' - url = url_for(endpoint, type_id=correlation_type, key_id=value) + endpoint = 'correlation.show_correlation' + url = url_for(endpoint, object_type="cryptocurrency", type_id=correlation_type, correlation_id=value) elif correlation_name == 'decoded': - endpoint = 'hashDecoded.showHash' - url = url_for(endpoint, hash=value) + endpoint = 'correlation.show_correlation' + url = url_for(endpoint, object_type="decoded", correlation_id=value) elif correlation_name == 'domain': endpoint = 'crawler_splash.showDomain' url = url_for(endpoint, domain=value) diff --git a/bin/lib/Decoded.py b/bin/lib/Decoded.py index 40ab9fd9..1618b034 100755 --- a/bin/lib/Decoded.py +++ b/bin/lib/Decoded.py @@ -8,6 +8,7 @@ import redis sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) import Item +import Date import ConfigLoader @@ -24,12 +25,54 @@ def get_decoded_item_type(sha1_string): ''' return r_serv_metadata.hget('metadata_hash:{}'.format(sha1_string), 'estimated_type') -def get_decoded_metadata(sha1_string): +def nb_decoded_seen_in_item(sha1_string): + nb = r_serv_metadata.hget('metadata_hash:{}'.format(sha1_string), 'nb_seen_in_all_pastes') + if nb is None: + return 0 + else: + return int(nb) + +def nb_decoded_item_size(sha1_string): + nb = r_serv_metadata.hget('metadata_hash:{}'.format(sha1_string), 'size') + if nb is None: + return 0 + else: + return int(nb) + +def get_decoded_metadata(sha1_string, nb_seen=False, size=False): metadata_dict = {} metadata_dict['first_seen'] = r_serv_metadata.hget('metadata_hash:{}'.format(sha1_string), 'first_seen') metadata_dict['last_seen'] = r_serv_metadata.hget('metadata_hash:{}'.format(sha1_string), 'last_seen') + if nb_seen: + metadata_dict['nb_seen'] = nb_decoded_seen_in_item(sha1_string) + if size: + metadata_dict['size'] = nb_decoded_item_size(sha1_string) return metadata_dict +def get_list_nb_previous_hash(sha1_string, num_day): + nb_previous_hash = [] + for date_day in Date.get_previous_date_list(num_day): + nb_previous_hash.append(get_nb_hash_seen_by_date(sha1_string, date_day)) + return nb_previous_hash + +def get_nb_hash_seen_by_date(sha1_string, date_day): + nb = r_serv_metadata.zscore('hash_date:{}'.format(date_day), sha1_string) + if nb is None: + return 0 + else: + return int(nb) + +def get_decoded_vt_report(sha1_string): + vt_dict = {} + res = r_serv_metadata.hget('metadata_hash:{}'.format(sha1_string), 'vt_link') + if res: + vt_dict["link"] = res + res = r_serv_metadata.hget('metadata_hash:{}'.format(sha1_string), 'vt_report') + if res: + vt_dict["report"] = res + return vt_dict + + def get_decoded_items_list(sha1_string): return r_serv_metadata.zrange('nb_seen_hash:{}'.format(sha1_string), 0, -1) diff --git a/bin/lib/Domain.py b/bin/lib/Domain.py index 5267839c..5b9310be 100755 --- a/bin/lib/Domain.py +++ b/bin/lib/Domain.py @@ -14,7 +14,6 @@ import redis import random sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) -import Correlation import Cryptocurrency from Pgp import pgp import Decoded @@ -25,6 +24,7 @@ cryptocurrency = Cryptocurrency.cryptocurrency sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader +import Correlate_object config_loader = ConfigLoader.ConfigLoader() r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") @@ -220,7 +220,7 @@ def get_domain_all_correlation(domain, correlation_names=[], get_nb=False): :rtype: dict ''' if not correlation_names: - correlation_names = Correlation.get_all_correlation_names() + correlation_names = Correlate_object.get_all_correlation_names() domain_correl = {} for correlation_name in correlation_names: if correlation_name=='cryptocurrency': diff --git a/bin/packages/Correlation.py b/bin/packages/Correlation.py index 3921163e..1f27b9a2 100755 --- a/bin/packages/Correlation.py +++ b/bin/packages/Correlation.py @@ -8,6 +8,9 @@ import redis sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) +import Date + config_loader = ConfigLoader.ConfigLoader() r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") config_loader = None @@ -35,8 +38,31 @@ class Correlation(object): meta_dict = {} meta_dict['first_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, correlation_type, field_name), 'first_seen') meta_dict['last_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, correlation_type, field_name), 'last_seen') + meta_dict['nb_seen'] = r_serv_metadata.scard('set_{}_{}:{}'.format(self.correlation_name, correlation_type, field_name)) return meta_dict + def get_metadata(self, correlation_type, field_name, date_format='str_date'): + meta_dict = self._get_metadata(correlation_type, field_name) + if date_format == "str_date": + if meta_dict['first_seen']: + meta_dict['first_seen'] = '{}/{}/{}'.format(meta_dict['first_seen'][0:4], meta_dict['first_seen'][4:6], meta_dict['first_seen'][6:8]) + if meta_dict['last_seen']: + meta_dict['last_seen'] = '{}/{}/{}'.format(meta_dict['last_seen'][0:4], meta_dict['last_seen'][4:6], meta_dict['last_seen'][6:8]) + return meta_dict + + def get_nb_object_seen_by_date(self, correlation_type, field_name, date_day): + nb = r_serv_metadata.hget('{}:{}:{}'.format(self.correlation_name, correlation_type, date_day), field_name) + if nb is None: + return 0 + else: + return int(nb) + + def get_list_nb_previous_correlation_object(self, correlation_type, field_name, numDay): + nb_previous_correlation = [] + for date_day in Date.get_previous_date_list(numDay): + nb_previous_correlation.append(self.get_nb_object_seen_by_date(correlation_type, field_name, date_day)) + return nb_previous_correlation + def _get_correlation_by_date(self, correlation_type, date): return r_serv_metadata.hkeys('{}:{}:{}'.format(self.correlation_name, correlation_type, date)) diff --git a/bin/packages/Date.py b/bin/packages/Date.py index c9997bab..bf05020e 100644 --- a/bin/packages/Date.py +++ b/bin/packages/Date.py @@ -2,6 +2,8 @@ import datetime +# # TODO: refractor me + class Date(object): """docstring for Date""" def __init__(self, *args): @@ -52,6 +54,7 @@ def date_substract_day(date, num_day=1): new_date = str(new_date).replace('-', '') return new_date +# # TODO: remove me ## FIXME: def get_date_range(num_day): curr_date = datetime.date.today() date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2)) @@ -61,6 +64,14 @@ def get_date_range(num_day): date_list.append(date.substract_day(i)) return list(reversed(date_list)) +def get_previous_date_list(num_day): + curr_date = datetime.date.today() + date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2)) + date_list = [] + for i in range(0, num_day+1): + date_list.append(date.substract_day(i)) + return list(reversed(date_list)) + def substract_date(date_from, date_to): date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8])) date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8])) diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py index 826f0809..ca2972de 100644 --- a/var/www/blueprints/correlation.py +++ b/var/www/blueprints/correlation.py @@ -23,32 +23,25 @@ from Role_Manager import login_admin, login_analyst sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) import Correlate_object +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) +import Cryptocurrency +import Pgp +import Decoded + bootstrap_label = Flask_config.bootstrap_label +vt_enabled = Flask_config.vt_enabled # ============ BLUEPRINT ============ correlation = Blueprint('correlation', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/correlation')) # ============ VARIABLES ============ -def show_correlation(correlation_type, type_id, key_id): - if is_valid_type_id(correlation_type, type_id): - key_id_metadata = get_key_id_metadata(correlation_type, type_id, key_id) - if key_id_metadata: - - num_day_sparkline = 6 - date_range_sparkline = get_date_range(num_day_sparkline) - - sparkline_values = list_sparkline_type_id_values(date_range_sparkline, correlation_type, type_id, key_id) - return render_template('show_correlation.html', key_id=key_id, type_id=type_id, - correlation_type=correlation_type, - graph_line_endpoint=get_graph_line_json_endpoint(correlation_type), - key_id_metadata=key_id_metadata, - type_icon=get_icon(correlation_type, type_id), - sparkline_values=sparkline_values) - else: - return '404' - else: - return 'error' +###### +### graph_line_json +### 'hashDecoded.pgpdump_graph_line_json' +### 'hashDecoded.cryptocurrency_graph_line_json' +### +###### # ============ FUNCTIONS ============ @@ -101,11 +94,30 @@ def sanitise_correlation_objects(correlation_objects): else: return all_correlation_objects +def get_card_metadata(object_type, correlation_id, type_id=None): + card_dict = {} + if object_type == 'cryptocurrency': + card_dict["sparkline"] = Cryptocurrency.cryptocurrency.get_list_nb_previous_correlation_object(type_id, correlation_id, 6) + card_dict["icon"] = Correlate_object.get_correlation_node_icon(object_type, type_id) + elif object_type == 'pgp': + card_dict["sparkline"] = Pgp.pgp.get_list_nb_previous_correlation_object(type_id, correlation_id, 6) + card_dict["icon"] = Correlate_object.get_correlation_node_icon(object_type, type_id) + elif object_type == 'decoded': + card_dict["sparkline"] = Decoded.get_list_nb_previous_hash(correlation_id, 6) + card_dict["icon"] = Correlate_object.get_correlation_node_icon(object_type, value=correlation_id) + card_dict["vt"] = Decoded.get_decoded_vt_report(correlation_id) + card_dict["vt"]["status"] = vt_enabled + elif object_type == 'domain': + pass + elif object_type == 'paste': + pass + return card_dict + # ============= ROUTES ============== @correlation.route('/correlation/show_correlation', methods=['GET', 'POST']) # GET + POST @login_required @login_analyst -def showCorrelationDomain(): +def show_correlation(): if request.method == 'POST': object_type = request.form.get('object_type') type_id = request.form.get('type_id') @@ -143,7 +155,7 @@ def showCorrelationDomain(): correlation_objects = ",".join(correlation_objects) # redirect to keep history and bookmark - return redirect(url_for('correlation.showCorrelationDomain', object_type=object_type, type_id=type_id, correlation_id=correlation_id, mode=mode, + return redirect(url_for('correlation.show_correlation', object_type=object_type, type_id=type_id, correlation_id=correlation_id, mode=mode, max_nodes=max_nodes, correlation_names=correlation_names, correlation_objects=correlation_objects)) # request.method == 'GET' @@ -167,6 +179,7 @@ def showCorrelationDomain(): dict_object["metadata"] = Correlate_object.get_object_metadata(object_type, correlation_id, type_id=type_id) if type_id: dict_object["metadata"]['type_id'] = type_id + dict_object["metadata_card"] = get_card_metadata(object_type, correlation_id, type_id=type_id) return render_template("show_correlation.html", dict_object=dict_object) @correlation.route('/correlation/graph_node_json') diff --git a/var/www/modules/hashDecoded/Flask_hashDecoded.py b/var/www/modules/hashDecoded/Flask_hashDecoded.py index ae3ee48b..76b398a9 100644 --- a/var/www/modules/hashDecoded/Flask_hashDecoded.py +++ b/var/www/modules/hashDecoded/Flask_hashDecoded.py @@ -224,13 +224,7 @@ def get_correlation_type_page_endpoint(correlation_type): return endpoint def get_show_key_id_endpoint(correlation_type): - if correlation_type == 'pgpdump': - endpoint = 'hashDecoded.show_pgpdump' - elif correlation_type == 'cryptocurrency': - endpoint = 'hashDecoded.show_cryptocurrency' - else: - endpoint = 'hashDecoded.hashDecoded_page' - return endpoint + return 'correlation.show_correlation' def get_range_type_json_endpoint(correlation_type): if correlation_type == 'pgpdump': @@ -351,8 +345,13 @@ def main_correlation_page(correlation_type, type_id, date_from, date_to, show_de l_type = get_all_types_id(correlation_type) + correlation_type_n = correlation_type + if correlation_type_n=='pgpdump': + correlation_type_n = 'pgp' + return render_template("DaysCorrelation.html", all_metadata=keys_id_metadata, correlation_type=correlation_type, + correlation_type_n=correlation_type_n, correlation_type_endpoint=get_correlation_type_page_endpoint(correlation_type), correlation_type_search_endpoint=get_correlation_type_search_endpoint(correlation_type), show_key_id_endpoint=get_show_key_id_endpoint(correlation_type), @@ -362,27 +361,27 @@ def main_correlation_page(correlation_type, type_id, date_from, date_to, show_de date_from=date_from, date_to=date_to, show_decoded_files=show_decoded_files) -def show_correlation(correlation_type, type_id, key_id): - if is_valid_type_id(correlation_type, type_id): - key_id_metadata = get_key_id_metadata(correlation_type, type_id, key_id) - if key_id_metadata: - - num_day_sparkline = 6 - date_range_sparkline = get_date_range(num_day_sparkline) - - sparkline_values = list_sparkline_type_id_values(date_range_sparkline, correlation_type, type_id, key_id) - return render_template('showCorrelation.html', key_id=key_id, type_id=type_id, - correlation_type=correlation_type, - graph_node_endpoint=get_graph_node_json_endpoint(correlation_type), - graph_line_endpoint=get_graph_line_json_endpoint(correlation_type), - font_family=get_font_family(correlation_type), - key_id_metadata=key_id_metadata, - type_icon=get_icon(correlation_type, type_id), - sparkline_values=sparkline_values) - else: - return '404' - else: - return 'error' +# def show_correlation(correlation_type, type_id, key_id): +# if is_valid_type_id(correlation_type, type_id): +# key_id_metadata = get_key_id_metadata(correlation_type, type_id, key_id) +# if key_id_metadata: +# +# num_day_sparkline = 6 +# date_range_sparkline = get_date_range(num_day_sparkline) +# +# sparkline_values = list_sparkline_type_id_values(date_range_sparkline, correlation_type, type_id, key_id) +# return render_template('showCorrelation.html', key_id=key_id, type_id=type_id, +# correlation_type=correlation_type, +# graph_node_endpoint=get_graph_node_json_endpoint(correlation_type), +# graph_line_endpoint=get_graph_line_json_endpoint(correlation_type), +# font_family=get_font_family(correlation_type), +# key_id_metadata=key_id_metadata, +# type_icon=get_icon(correlation_type, type_id), +# sparkline_values=sparkline_values) +# else: +# return '404' +# else: +# return 'error' def correlation_type_range_type_json(correlation_type, date_from, date_to): date_range = [] @@ -620,60 +619,60 @@ def hash_hash(): hash = request.args.get('hash') return render_template('hash_hash.html') - -@hashDecoded.route('/hashDecoded/showHash') -@login_required -@login_analyst -def showHash(): - hash = request.args.get('hash') - #hash = 'e02055d3efaad5d656345f6a8b1b6be4fe8cb5ea' - - # TODO FIXME show error - if hash is None: - return hashDecoded_page() - - estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type') - # hash not found - # TODO FIXME show error - if estimated_type is None: - return hashDecoded_page() - - else: - file_icon = get_file_icon(estimated_type) - size = r_serv_metadata.hget('metadata_hash:'+hash, 'size') - first_seen = r_serv_metadata.hget('metadata_hash:'+hash, 'first_seen') - last_seen = r_serv_metadata.hget('metadata_hash:'+hash, 'last_seen') - nb_seen_in_all_pastes = r_serv_metadata.hget('metadata_hash:'+hash, 'nb_seen_in_all_pastes') - - # get all encoding for this hash - list_hash_decoder = [] - list_decoder = r_serv_metadata.smembers('all_decoder') - for decoder in list_decoder: - encoding = r_serv_metadata.hget('metadata_hash:'+hash, decoder+'_decoder') - if encoding is not None: - list_hash_decoder.append({'encoding': decoder, 'nb_seen': encoding}) - - num_day_type = 6 - date_range_sparkline = get_date_range(num_day_type) - sparkline_values = list_sparkline_values(date_range_sparkline, hash) - - if r_serv_metadata.hexists('metadata_hash:'+hash, 'vt_link'): - b64_vt = True - b64_vt_link = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_link') - b64_vt_report = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_report') - else: - b64_vt = False - b64_vt_link = '' - b64_vt_report = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_report') - # hash never refreshed - if b64_vt_report is None: - b64_vt_report = '' - - return render_template('showHash.html', hash=hash, vt_enabled=vt_enabled, b64_vt=b64_vt, b64_vt_link=b64_vt_link, - b64_vt_report=b64_vt_report, - size=size, estimated_type=estimated_type, file_icon=file_icon, - first_seen=first_seen, list_hash_decoder=list_hash_decoder, - last_seen=last_seen, nb_seen_in_all_pastes=nb_seen_in_all_pastes, sparkline_values=sparkline_values) +# +# @hashDecoded.route('/hashDecoded/showHash') +# @login_required +# @login_analyst +# def showHash(): +# hash = request.args.get('hash') +# #hash = 'e02055d3efaad5d656345f6a8b1b6be4fe8cb5ea' +# +# # TODO FIXME show error +# if hash is None: +# return hashDecoded_page() +# +# estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type') +# # hash not found +# # TODO FIXME show error +# if estimated_type is None: +# return hashDecoded_page() +# +# else: +# file_icon = get_file_icon(estimated_type) +# size = r_serv_metadata.hget('metadata_hash:'+hash, 'size') +# first_seen = r_serv_metadata.hget('metadata_hash:'+hash, 'first_seen') +# last_seen = r_serv_metadata.hget('metadata_hash:'+hash, 'last_seen') +# nb_seen_in_all_pastes = r_serv_metadata.hget('metadata_hash:'+hash, 'nb_seen_in_all_pastes') +# +# # get all encoding for this hash +# list_hash_decoder = [] +# list_decoder = r_serv_metadata.smembers('all_decoder') +# for decoder in list_decoder: +# encoding = r_serv_metadata.hget('metadata_hash:'+hash, decoder+'_decoder') +# if encoding is not None: +# list_hash_decoder.append({'encoding': decoder, 'nb_seen': encoding}) +# +# num_day_type = 6 +# date_range_sparkline = get_date_range(num_day_type) +# sparkline_values = list_sparkline_values(date_range_sparkline, hash) +# +# if r_serv_metadata.hexists('metadata_hash:'+hash, 'vt_link'): +# b64_vt = True +# b64_vt_link = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_link') +# b64_vt_report = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_report') +# else: +# b64_vt = False +# b64_vt_link = '' +# b64_vt_report = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_report') +# # hash never refreshed +# if b64_vt_report is None: +# b64_vt_report = '' +# +# return render_template('showHash.html', hash=hash, vt_enabled=vt_enabled, b64_vt=b64_vt, b64_vt_link=b64_vt_link, +# b64_vt_report=b64_vt_report, +# size=size, estimated_type=estimated_type, file_icon=file_icon, +# first_seen=first_seen, list_hash_decoder=list_hash_decoder, +# last_seen=last_seen, nb_seen_in_all_pastes=nb_seen_in_all_pastes, sparkline_values=sparkline_values) @hashDecoded.route('/hashDecoded/downloadHash') @@ -1207,22 +1206,22 @@ def all_cryptocurrency_search(): show_decoded_files = request.form.get('show_decoded_files') return redirect(url_for('hashDecoded.cryptocurrency_page', date_from=date_from, date_to=date_to, type_id=type_id, show_decoded_files=show_decoded_files)) -@hashDecoded.route('/correlation/show_pgpdump') -@login_required -@login_analyst -def show_pgpdump(): - type_id = request.args.get('type_id') - key_id = request.args.get('key_id') - return show_correlation('pgpdump', type_id, key_id) - - -@hashDecoded.route('/correlation/show_cryptocurrency') -@login_required -@login_analyst -def show_cryptocurrency(): - type_id = request.args.get('type_id') - key_id = request.args.get('key_id') - return show_correlation('cryptocurrency', type_id, key_id) +# @hashDecoded.route('/correlation/show_pgpdump') +# @login_required +# @login_analyst +# def show_pgpdump(): +# type_id = request.args.get('type_id') +# key_id = request.args.get('key_id') +# return show_correlation('pgpdump', type_id, key_id) +# +# +# @hashDecoded.route('/correlation/show_cryptocurrency') +# @login_required +# @login_analyst +# def show_cryptocurrency(): +# type_id = request.args.get('type_id') +# key_id = request.args.get('key_id') +# return show_correlation('cryptocurrency', type_id, key_id) @hashDecoded.route('/correlation/cryptocurrency_range_type_json') @login_required @@ -1248,6 +1247,7 @@ def pgpdump_graph_node_json(): key_id = request.args.get('key_id') return correlation_graph_node_json('pgpdump', type_id, key_id) +# # TODO: REFRACTOR @hashDecoded.route('/correlation/cryptocurrency_graph_node_json') @login_required @login_analyst @@ -1256,6 +1256,7 @@ def cryptocurrency_graph_node_json(): key_id = request.args.get('key_id') return correlation_graph_node_json('cryptocurrency', type_id, key_id) +# # TODO: REFRACTOR @hashDecoded.route('/correlation/pgpdump_graph_line_json') @login_required @login_analyst diff --git a/var/www/modules/hashDecoded/templates/DaysCorrelation.html b/var/www/modules/hashDecoded/templates/DaysCorrelation.html index 9de55d5d..f3b5235f 100644 --- a/var/www/modules/hashDecoded/templates/DaysCorrelation.html +++ b/var/www/modules/hashDecoded/templates/DaysCorrelation.html @@ -145,7 +145,7 @@ {% for key_id in all_metadata %}   {{ all_metadata[key_id]['type_id'] }} - {{ key_id }} + {{ key_id }} {{ all_metadata[key_id]['first_seen'] }} {{ all_metadata[key_id]['last_seen'] }} {{ all_metadata[key_id]['nb_seen'] }} @@ -224,9 +224,6 @@ chart.stackBarChart = barchart_type_stack("{{ url_for(range_type_json_endpoint) }}?date_from={{date_from}}&date_to={{date_to}}", 'id'); {% endif %} - //draw_pie_chart("pie_chart_encoded" ,"{{ url_for('hashDecoded.decoder_type_json') }}?date_from={{date_from}}&date_to={{date_to}}&type={{type}}", "{{ url_for('hashDecoded.hashDecoded_page') }}?date_from={{date_from}}&date_to={{date_to}}&type={{type}}&encoding="); - //draw_pie_chart("pie_chart_top5_types" ,"{{ url_for('hashDecoded.top5_type_json') }}?date_from={{date_from}}&date_to={{date_to}}&type={{type}}", "{{ url_for('hashDecoded.hashDecoded_page') }}?date_from={{date_from}}&date_to={{date_to}}&type="); - chart.onResize(); $(window).on("resize", function() { chart.onResize(); diff --git a/var/www/modules/hashDecoded/templates/hashDecoded.html b/var/www/modules/hashDecoded/templates/hashDecoded.html index fdd2ace1..92b89b7e 100644 --- a/var/www/modules/hashDecoded/templates/hashDecoded.html +++ b/var/www/modules/hashDecoded/templates/hashDecoded.html @@ -158,7 +158,7 @@ {% for b64 in l_64 %}   {{ b64[1] }} - {{ b64[2] }} + {{ b64[2] }} {{ b64[5] }} {{ b64[6] }} {{ b64[3] }} diff --git a/var/www/modules/hashDecoded/templates/showCorrelation.html b/var/www/modules/hashDecoded/templates/showCorrelation.html deleted file mode 100644 index a2f4d827..00000000 --- a/var/www/modules/hashDecoded/templates/showCorrelation.html +++ /dev/null @@ -1,512 +0,0 @@ - - - - - - - AIL - framework - - - - - - - - - - - - - - - - - - {% include 'nav_bar.html' %} - -
-
- - {% include 'decoded/menu_sidebar.html' %} - -
- -
-
-

{{ key_id }} :

-
    -
  • -
    -
    - - - - - - - - - - - - - - - - - -
    typeFirst_seenLast_seenNb seen
      {{ type_id }}{{ key_id_metadata['first_seen'] }}{{ key_id_metadata['last_seen'] }}{{ key_id_metadata['nb_seen'] }}
    -
    -
    -
    -
    -
    -
    • -
-
-
- -
-
- -
-
- Graph -
-
-
-
-
-
-
- -
- -
-
- Graph -
-
- - - {% if correlation_type=='pgpdump' %} - {% include 'decoded/show_helper_pgpdump.html' %} - {% elif correlation_type=='cryptocurrency' %} - {% include 'decoded/show_helper_cryptocurrency.html' %} - {% endif %} - -
-
-
-
- -
-
- Graph -
-
-
-
-
-
- -
-
-
- - - - - - - - - - - - diff --git a/var/www/modules/hashDecoded/templates/showHash.html b/var/www/modules/hashDecoded/templates/showHash.html deleted file mode 100644 index b6700557..00000000 --- a/var/www/modules/hashDecoded/templates/showHash.html +++ /dev/null @@ -1,587 +0,0 @@ - - - - - - - Hash Information - AIL - - - - - - - - - - - - - - - - - - {% include 'nav_bar.html' %} - -
-
- - {% include 'decoded/menu_sidebar.html' %} - -
- -
-
-

{{ hash }} :

-
    -
  • -
    -
    - - - - - - - - - - - - - - - - - - - -
    Estimated typeFirst_seenLast_seenSize (Kb)Nb seen
      {{ estimated_type }}{{ first_seen }}{{ last_seen }}{{ size }}{{ nb_seen_in_all_pastes }}
    -
    -
    -
    -
    -
    -
    • -
- - {% if vt_enabled %} - {% if not b64_vt %} - - - - {% else %} - VT Report - {% endif %} - - {% else %} - Virus Total submission is disabled - {% endif %} - - - - -
-
- -
-
- -
-
- Graph -
-
-
-
-
-
-
- -
- -
-
- Encoding -
-
- {% for encoding in list_hash_decoder %} - - {% endfor %} -
-
- -
-
- Graph -
-
- - - -
    -
    • -
  • -

    Double click on a node to open Hash/Paste

    - - Current Hash
    - - Hashes
    - - Pastes -

    -
    • -
  • - Hash Types: -
    • -
  • - Application
    - Audio
    - Image
    - Text
    - Other -
    • -
-
-
-
-
- -
-
- Graph -
-
-
-
-
-
- -
-
-
- - - - - - - - - - - - - diff --git a/var/www/templates/correlation/legend_graph_correlation.html b/var/www/templates/correlation/legend_graph_correlation.html new file mode 100644 index 00000000..9d562b58 --- /dev/null +++ b/var/www/templates/correlation/legend_graph_correlation.html @@ -0,0 +1,168 @@ + + + + + + + + + + + + + + + + + + + +
+ Cryptocurrency: + + Decoded: + + Pgp: + + Domain: + + Paste: +
+
+ + + + + + + bitcoin +
+
+ + + + + + + monero +
+
+ + + + + + + etherum +
+
+ + + + + + + other cryptocurrencies +
+ 		+
+ + + + + + + application +
+
+ + + + + + + audio +
+
+ + + + + + + text +
+
+ + + + + + + other types of file +
+ 		+
+ + + + + + + key +
+
+ + + + + + + name +
+
+ + + + + + + mail +
+ 		+
+ + + + + + + onion +
+
+ + + + + + + web +
+ 		+
+ + + + + + crawled +
+
+ + + + + + other +
+
diff --git a/var/www/templates/correlation/metadata_card_decoded.html b/var/www/templates/correlation/metadata_card_decoded.html new file mode 100644 index 00000000..fe19bb51 --- /dev/null +++ b/var/www/templates/correlation/metadata_card_decoded.html @@ -0,0 +1,186 @@ +
+
+

{{ dict_object["correlation_id"] }} :

+
    +
  • +
    +
    + + + + + + + + + + + + + + + + + + + + + +
    Object typeEstimated typeFirst seenLast seenSize (Kb)Nb seen
    {{ dict_object["object_type"] }} + + + + {{ dict_object["metadata_card"]["icon"]["icon_text"] }} + + + {{ dict_object["metadata"]["type_id"] }} + {{ dict_object["metadata"]['first_seen'] }}{{ dict_object["metadata"]['last_seen'] }}{{ dict_object["metadata"]['size'] }}{{ dict_object["metadata"]['nb_seen'] }}
    +
    +
    +
    +
    +
    +
    • +
+ + {% if dict_object["metadata_card"]["vt"]["status"] %} + {% if not "link" in dict_object["metadata_card"]["vt"] %} + + + + {% else %} + VT Report + {% endif %} + + {% else %} + Virus Total submission is disabled + {% endif %} + + + + +
+
+ + + + + + + diff --git a/var/www/templates/correlation/metadata_card_pgp.html b/var/www/templates/correlation/metadata_card_pgp.html new file mode 100644 index 00000000..25815058 --- /dev/null +++ b/var/www/templates/correlation/metadata_card_pgp.html @@ -0,0 +1,140 @@ +
+
+

{{ dict_object["correlation_id"] }} :

+
    +
  • +
    +
    + + + + + + + + + + + + + + + + + + + +
    Object typetypeFirst seenLast seenNb seen
    {{ dict_object["object_type"] }} + + + + {{ dict_object["metadata_card"]["icon"]["icon_text"] }} + + + {{ dict_object["metadata"]["type_id"] }} + {{ dict_object["metadata"]['first_seen'] }}{{ dict_object["metadata"]['last_seen'] }}{{ dict_object["metadata"]['nb_seen'] }}
    +
    +
    +
    +
    +
    +
    • +
+
+
+ + + + + diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index 1cf70f67..f6e00ae3 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -18,6 +18,11 @@